services/crunch-run/crunchrun_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bufio"
   9         "bytes"
  10         "context"
  11         "crypto/md5"
  12         "encoding/json"
  13         "errors"
  14         "fmt"
  15         "io"
  16         "io/ioutil"
  17         "net"
  18         "os"
  19         "os/exec"
  20         "path/filepath"
  21         "runtime/pprof"
  22         "sort"
  23         "strings"
  24         "sync"
  25         "syscall"
  26         "testing"
  27         "time"
  28
  29         "git.curoverse.com/arvados.git/sdk/go/arvados"
  30         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  31         "git.curoverse.com/arvados.git/sdk/go/manifest"
  32
  33         dockertypes "github.com/docker/docker/api/types"
  34         dockercontainer "github.com/docker/docker/api/types/container"
  35         dockernetwork "github.com/docker/docker/api/types/network"
  36         . "gopkg.in/check.v1"
  37 )
  38
  39 // Gocheck boilerplate
  40 func TestCrunchExec(t *testing.T) {
  41         TestingT(t)
  42 }
  43
  44 type TestSuite struct{}
  45
  46 // Gocheck boilerplate
  47 var _ = Suite(&TestSuite{})
  48
  49 type ArvTestClient struct {
  50         Total   int64
  51         Calls   int
  52         Content []arvadosclient.Dict
  53         arvados.Container
  54         Logs map[string]*bytes.Buffer
  55         sync.Mutex
  56         WasSetRunning bool
  57         callraw       bool
  58 }
  59
  60 type KeepTestClient struct {
  61         Called  bool
  62         Content []byte
  63 }
  64
  65 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
  66 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
  67 var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
  68
  69 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
  70 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
  71
  72 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
  73 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
  74 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
  75 `
  76
  77 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
  78
  79 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
  80 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
  81
  82 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
  83 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
  84
  85 type TestDockerClient struct {
  86         imageLoaded string
  87         logReader   io.ReadCloser
  88         logWriter   io.WriteCloser
  89         fn          func(t *TestDockerClient)
  90         finish      int
  91         stop        chan bool
  92         cwd         string
  93         env         []string
  94         api         *ArvTestClient
  95         realTemp    string
  96 }
  97
  98 func NewTestDockerClient(exitCode int) *TestDockerClient {
  99         t := &TestDockerClient{}
 100         t.logReader, t.logWriter = io.Pipe()
 101         t.finish = exitCode
 102         t.stop = make(chan bool, 1)
 103         t.cwd = "/"
 104         return t
 105 }
 106
 107 type MockConn struct {
 108         net.Conn
 109 }
 110
 111 func (m *MockConn) Write(b []byte) (int, error) {
 112         return len(b), nil
 113 }
 114
 115 func NewMockConn() *MockConn {
 116         c := &MockConn{}
 117         return c
 118 }
 119
 120 func (t *TestDockerClient) ContainerAttach(ctx context.Context, container string, options dockertypes.ContainerAttachOptions) (dockertypes.HijackedResponse, error) {
 121         return dockertypes.HijackedResponse{Conn: NewMockConn(), Reader: bufio.NewReader(t.logReader)}, nil
 122 }
 123
 124 func (t *TestDockerClient) ContainerCreate(ctx context.Context, config *dockercontainer.Config, hostConfig *dockercontainer.HostConfig, networkingConfig *dockernetwork.NetworkingConfig, containerName string) (dockercontainer.ContainerCreateCreatedBody, error) {
 125         if config.WorkingDir != "" {
 126                 t.cwd = config.WorkingDir
 127         }
 128         t.env = config.Env
 129         return dockercontainer.ContainerCreateCreatedBody{ID: "abcde"}, nil
 130 }
 131
 132 func (t *TestDockerClient) ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error {
 133         if t.finish == 3 {
 134                 return errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
 135         }
 136         if t.finish == 4 {
 137                 return errors.New(`panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`)
 138         }
 139         if t.finish == 5 {
 140                 return errors.New(`Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`)
 141         }
 142         if t.finish == 6 {
 143                 return errors.New(`Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`)
 144         }
 145
 146         if container == "abcde" {
 147                 // t.fn gets executed in ContainerWait
 148                 return nil
 149         } else {
 150                 return errors.New("Invalid container id")
 151         }
 152 }
 153
 154 func (t *TestDockerClient) ContainerStop(ctx context.Context, container string, timeout *time.Duration) error {
 155         t.stop <- true
 156         return nil
 157 }
 158
 159 func (t *TestDockerClient) ContainerWait(ctx context.Context, container string, condition dockercontainer.WaitCondition) (<-chan dockercontainer.ContainerWaitOKBody, <-chan error) {
 160         body := make(chan dockercontainer.ContainerWaitOKBody)
 161         err := make(chan error)
 162         go func() {
 163                 t.fn(t)
 164                 body <- dockercontainer.ContainerWaitOKBody{StatusCode: int64(t.finish)}
 165                 close(body)
 166                 close(err)
 167         }()
 168         return body, err
 169 }
 170
 171 func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
 172         if t.finish == 2 {
 173                 return dockertypes.ImageInspect{}, nil, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
 174         }
 175
 176         if t.imageLoaded == image {
 177                 return dockertypes.ImageInspect{}, nil, nil
 178         } else {
 179                 return dockertypes.ImageInspect{}, nil, errors.New("")
 180         }
 181 }
 182
 183 func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) {
 184         if t.finish == 2 {
 185                 return dockertypes.ImageLoadResponse{}, fmt.Errorf("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
 186         }
 187         _, err := io.Copy(ioutil.Discard, input)
 188         if err != nil {
 189                 return dockertypes.ImageLoadResponse{}, err
 190         } else {
 191                 t.imageLoaded = hwImageId
 192                 return dockertypes.ImageLoadResponse{Body: ioutil.NopCloser(input)}, nil
 193         }
 194 }
 195
 196 func (*TestDockerClient) ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error) {
 197         return nil, nil
 198 }
 199
 200 func (client *ArvTestClient) Create(resourceType string,
 201         parameters arvadosclient.Dict,
 202         output interface{}) error {
 203
 204         client.Mutex.Lock()
 205         defer client.Mutex.Unlock()
 206
 207         client.Calls++
 208         client.Content = append(client.Content, parameters)
 209
 210         if resourceType == "logs" {
 211                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 212                 if client.Logs == nil {
 213                         client.Logs = make(map[string]*bytes.Buffer)
 214                 }
 215                 if client.Logs[et] == nil {
 216                         client.Logs[et] = &bytes.Buffer{}
 217                 }
 218                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 219         }
 220
 221         if resourceType == "collections" && output != nil {
 222                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 223                 outmap := output.(*arvados.Collection)
 224                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 225         }
 226
 227         return nil
 228 }
 229
 230 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 231         switch {
 232         case method == "GET" && resourceType == "containers" && action == "auth":
 233                 return json.Unmarshal([]byte(`{
 234                         "kind": "arvados#api_client_authorization",
 235                         "uuid": "`+fakeAuthUUID+`",
 236                         "api_token": "`+fakeAuthToken+`"
 237                         }`), output)
 238         default:
 239                 return fmt.Errorf("Not found")
 240         }
 241 }
 242
 243 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 244         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 245         var j []byte
 246         if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
 247                 j, err = json.Marshal(client.Container)
 248         } else {
 249                 j = []byte(`{
 250                         "command": ["sleep", "1"],
 251                         "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 252                         "cwd": ".",
 253                         "environment": {},
 254                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
 255                         "output_path": "/tmp",
 256                         "priority": 1,
 257                         "runtime_constraints": {}
 258                 }`)
 259         }
 260         return ioutil.NopCloser(bytes.NewReader(j)), err
 261 }
 262
 263 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 264         if resourceType == "collections" {
 265                 if uuid == hwPDH {
 266                         output.(*arvados.Collection).ManifestText = hwManifest
 267                 } else if uuid == otherPDH {
 268                         output.(*arvados.Collection).ManifestText = otherManifest
 269                 } else if uuid == normalizedWithSubdirsPDH {
 270                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 271                 } else if uuid == denormalizedWithSubdirsPDH {
 272                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 273                 }
 274         }
 275         if resourceType == "containers" {
 276                 (*output.(*arvados.Container)) = client.Container
 277         }
 278         return nil
 279 }
 280
 281 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 282         client.Mutex.Lock()
 283         defer client.Mutex.Unlock()
 284         client.Calls++
 285         client.Content = append(client.Content, parameters)
 286         if resourceType == "containers" {
 287                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 288                         client.WasSetRunning = true
 289                 }
 290         }
 291         return nil
 292 }
 293
 294 var discoveryMap = map[string]interface{}{
 295         "defaultTrashLifetime":               float64(1209600),
 296         "crunchLimitLogBytesPerJob":          float64(67108864),
 297         "crunchLogThrottleBytes":             float64(65536),
 298         "crunchLogThrottlePeriod":            float64(60),
 299         "crunchLogThrottleLines":             float64(1024),
 300         "crunchLogPartialLineThrottlePeriod": float64(5),
 301         "crunchLogBytesPerEvent":             float64(4096),
 302         "crunchLogSecondsBetweenEvents":      float64(1),
 303 }
 304
 305 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 306         return discoveryMap[key], nil
 307 }
 308
 309 // CalledWith returns the parameters from the first API call whose
 310 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 311 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 312 // no call matches, it returns nil.
 313 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 314 call:
 315         for _, content := range client.Content {
 316                 var v interface{} = content
 317                 for _, k := range strings.Split(jpath, ".") {
 318                         if dict, ok := v.(arvadosclient.Dict); !ok {
 319                                 continue call
 320                         } else {
 321                                 v = dict[k]
 322                         }
 323                 }
 324                 if v == expect {
 325                         return content
 326                 }
 327         }
 328         return nil
 329 }
 330
 331 func (client *KeepTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 332         client.Content = buf
 333         return fmt.Sprintf("%s+%d", hash, len(buf)), len(buf), nil
 334 }
 335
 336 func (*KeepTestClient) ClearBlockCache() {
 337 }
 338
 339 type FileWrapper struct {
 340         io.ReadCloser
 341         len int64
 342 }
 343
 344 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
 345         return nil, errors.New("not implemented")
 346 }
 347
 348 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 349         return 0, errors.New("not implemented")
 350 }
 351
 352 func (fw FileWrapper) Size() int64 {
 353         return fw.len
 354 }
 355
 356 func (fw FileWrapper) Stat() (os.FileInfo, error) {
 357         return nil, errors.New("not implemented")
 358 }
 359
 360 func (fw FileWrapper) Truncate(int64) error {
 361         return errors.New("not implemented")
 362 }
 363
 364 func (fw FileWrapper) Write([]byte) (int, error) {
 365         return 0, errors.New("not implemented")
 366 }
 367
 368 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 369         if filename == hwImageId+".tar" {
 370                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 371                 client.Called = true
 372                 return FileWrapper{rdr, 1321984}, nil
 373         } else if filename == "/file1_in_main.txt" {
 374                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 375                 client.Called = true
 376                 return FileWrapper{rdr, 3}, nil
 377         }
 378         return nil, nil
 379 }
 380
 381 func (s *TestSuite) TestLoadImage(c *C) {
 382         kc := &KeepTestClient{}
 383         docker := NewTestDockerClient(0)
 384         cr := NewContainerRunner(&ArvTestClient{}, kc, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 385
 386         _, err := cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 387
 388         _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
 389         c.Check(err, NotNil)
 390
 391         cr.Container.ContainerImage = hwPDH
 392
 393         // (1) Test loading image from keep
 394         c.Check(kc.Called, Equals, false)
 395         c.Check(cr.ContainerConfig.Image, Equals, "")
 396
 397         err = cr.LoadImage()
 398
 399         c.Check(err, IsNil)
 400         defer func() {
 401                 cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 402         }()
 403
 404         c.Check(kc.Called, Equals, true)
 405         c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
 406
 407         _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
 408         c.Check(err, IsNil)
 409
 410         // (2) Test using image that's already loaded
 411         kc.Called = false
 412         cr.ContainerConfig.Image = ""
 413
 414         err = cr.LoadImage()
 415         c.Check(err, IsNil)
 416         c.Check(kc.Called, Equals, false)
 417         c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
 418
 419 }
 420
 421 type ArvErrorTestClient struct{}
 422
 423 func (ArvErrorTestClient) Create(resourceType string,
 424         parameters arvadosclient.Dict,
 425         output interface{}) error {
 426         return nil
 427 }
 428
 429 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 430         return errors.New("ArvError")
 431 }
 432
 433 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 434         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 435         return nil, errors.New("ArvError")
 436 }
 437
 438 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 439         return errors.New("ArvError")
 440 }
 441
 442 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 443         return nil
 444 }
 445
 446 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 447         return discoveryMap[key], nil
 448 }
 449
 450 type KeepErrorTestClient struct{}
 451
 452 func (KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 453         return "", 0, errors.New("KeepError")
 454 }
 455
 456 func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 457         return nil, errors.New("KeepError")
 458 }
 459
 460 func (KeepErrorTestClient) ClearBlockCache() {
 461 }
 462
 463 type KeepReadErrorTestClient struct{}
 464
 465 func (KeepReadErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 466         return "", 0, nil
 467 }
 468
 469 func (KeepReadErrorTestClient) ClearBlockCache() {
 470 }
 471
 472 type ErrorReader struct {
 473         FileWrapper
 474 }
 475
 476 func (ErrorReader) Read(p []byte) (n int, err error) {
 477         return 0, errors.New("ErrorReader")
 478 }
 479
 480 func (ErrorReader) Seek(int64, int) (int64, error) {
 481         return 0, errors.New("ErrorReader")
 482 }
 483
 484 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
 485         return ErrorReader{}, nil
 486 }
 487
 488 func (s *TestSuite) TestLoadImageArvError(c *C) {
 489         // (1) Arvados error
 490         cr := NewContainerRunner(ArvErrorTestClient{}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 491         cr.Container.ContainerImage = hwPDH
 492
 493         err := cr.LoadImage()
 494         c.Check(err.Error(), Equals, "While getting container image collection: ArvError")
 495 }
 496
 497 func (s *TestSuite) TestLoadImageKeepError(c *C) {
 498         // (2) Keep error
 499         docker := NewTestDockerClient(0)
 500         cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 501         cr.Container.ContainerImage = hwPDH
 502
 503         err := cr.LoadImage()
 504         c.Check(err.Error(), Equals, "While creating ManifestFileReader for container image: KeepError")
 505 }
 506
 507 func (s *TestSuite) TestLoadImageCollectionError(c *C) {
 508         // (3) Collection doesn't contain image
 509         cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 510         cr.Container.ContainerImage = otherPDH
 511
 512         err := cr.LoadImage()
 513         c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
 514 }
 515
 516 func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
 517         // (4) Collection doesn't contain image
 518         docker := NewTestDockerClient(0)
 519         cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 520         cr.Container.ContainerImage = hwPDH
 521
 522         err := cr.LoadImage()
 523         c.Check(err, NotNil)
 524 }
 525
 526 type ClosableBuffer struct {
 527         bytes.Buffer
 528 }
 529
 530 func (*ClosableBuffer) Close() error {
 531         return nil
 532 }
 533
 534 type TestLogs struct {
 535         Stdout ClosableBuffer
 536         Stderr ClosableBuffer
 537 }
 538
 539 func (tl *TestLogs) NewTestLoggingWriter(logstr string) io.WriteCloser {
 540         if logstr == "stdout" {
 541                 return &tl.Stdout
 542         }
 543         if logstr == "stderr" {
 544                 return &tl.Stderr
 545         }
 546         return nil
 547 }
 548
 549 func dockerLog(fd byte, msg string) []byte {
 550         by := []byte(msg)
 551         header := make([]byte, 8+len(by))
 552         header[0] = fd
 553         header[7] = byte(len(by))
 554         copy(header[8:], by)
 555         return header
 556 }
 557
 558 func (s *TestSuite) TestRunContainer(c *C) {
 559         docker := NewTestDockerClient(0)
 560         docker.fn = func(t *TestDockerClient) {
 561                 t.logWriter.Write(dockerLog(1, "Hello world\n"))
 562                 t.logWriter.Close()
 563         }
 564         cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 565
 566         var logs TestLogs
 567         cr.NewLogWriter = logs.NewTestLoggingWriter
 568         cr.Container.ContainerImage = hwPDH
 569         cr.Container.Command = []string{"./hw"}
 570         err := cr.LoadImage()
 571         c.Check(err, IsNil)
 572
 573         err = cr.CreateContainer()
 574         c.Check(err, IsNil)
 575
 576         err = cr.StartContainer()
 577         c.Check(err, IsNil)
 578
 579         err = cr.WaitFinish()
 580         c.Check(err, IsNil)
 581
 582         c.Check(strings.HasSuffix(logs.Stdout.String(), "Hello world\n"), Equals, true)
 583         c.Check(logs.Stderr.String(), Equals, "")
 584 }
 585
 586 func (s *TestSuite) TestCommitLogs(c *C) {
 587         api := &ArvTestClient{}
 588         kc := &KeepTestClient{}
 589         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 590         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 591
 592         cr.CrunchLog.Print("Hello world!")
 593         cr.CrunchLog.Print("Goodbye")
 594         cr.finalState = "Complete"
 595
 596         err := cr.CommitLogs()
 597         c.Check(err, IsNil)
 598
 599         c.Check(api.Calls, Equals, 2)
 600         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 601         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 602         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 603         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 604 }
 605
 606 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 607         api := &ArvTestClient{}
 608         kc := &KeepTestClient{}
 609         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 610
 611         err := cr.UpdateContainerRunning()
 612         c.Check(err, IsNil)
 613
 614         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 615 }
 616
 617 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 618         api := &ArvTestClient{}
 619         kc := &KeepTestClient{}
 620         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 621
 622         cr.LogsPDH = new(string)
 623         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 624
 625         cr.ExitCode = new(int)
 626         *cr.ExitCode = 42
 627         cr.finalState = "Complete"
 628
 629         err := cr.UpdateContainerFinal()
 630         c.Check(err, IsNil)
 631
 632         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 633         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 634         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 635 }
 636
 637 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 638         api := &ArvTestClient{}
 639         kc := &KeepTestClient{}
 640         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 641         cr.cCancelled = true
 642         cr.finalState = "Cancelled"
 643
 644         err := cr.UpdateContainerFinal()
 645         c.Check(err, IsNil)
 646
 647         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 648         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 649         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 650 }
 651
 652 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 653 // dress rehearsal of the Run() function, starting from a JSON container record.
 654 func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
 655         rec := arvados.Container{}
 656         err := json.Unmarshal([]byte(record), &rec)
 657         c.Check(err, IsNil)
 658
 659         docker := NewTestDockerClient(exitCode)
 660         docker.fn = fn
 661         docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 662
 663         api = &ArvTestClient{Container: rec}
 664         docker.api = api
 665         cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 666         cr.statInterval = 100 * time.Millisecond
 667         am := &ArvMountCmdLine{}
 668         cr.RunArvMount = am.ArvMountTest
 669
 670         realTemp, err = ioutil.TempDir("", "crunchrun_test1-")
 671         c.Assert(err, IsNil)
 672         defer os.RemoveAll(realTemp)
 673
 674         docker.realTemp = realTemp
 675
 676         tempcount := 0
 677         cr.MkTempDir = func(_ string, prefix string) (string, error) {
 678                 tempcount++
 679                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 680                 err := os.Mkdir(d, os.ModePerm)
 681                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 682                         // Test case must have pre-populated the tempdir
 683                         err = nil
 684                 }
 685                 return d, err
 686         }
 687
 688         if extraMounts != nil && len(extraMounts) > 0 {
 689                 err := cr.SetupArvMountPoint("keep")
 690                 c.Check(err, IsNil)
 691
 692                 for _, m := range extraMounts {
 693                         os.MkdirAll(cr.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 694                 }
 695         }
 696
 697         err = cr.Run()
 698         if api.CalledWith("container.state", "Complete") != nil {
 699                 c.Check(err, IsNil)
 700         }
 701         if exitCode != 2 {
 702                 c.Check(api.WasSetRunning, Equals, true)
 703                 c.Check(api.Content[api.Calls-2]["container"].(arvadosclient.Dict)["log"], NotNil)
 704         }
 705
 706         if err != nil {
 707                 for k, v := range api.Logs {
 708                         c.Log(k)
 709                         c.Log(v.String())
 710                 }
 711         }
 712
 713         return
 714 }
 715
 716 func (s *TestSuite) TestFullRunHello(c *C) {
 717         api, _, _ := FullRunHelper(c, `{
 718     "command": ["echo", "hello world"],
 719     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 720     "cwd": ".",
 721     "environment": {},
 722     "mounts": {"/tmp": {"kind": "tmp"} },
 723     "output_path": "/tmp",
 724     "priority": 1,
 725     "runtime_constraints": {}
 726 }`, nil, 0, func(t *TestDockerClient) {
 727                 t.logWriter.Write(dockerLog(1, "hello world\n"))
 728                 t.logWriter.Close()
 729         })
 730
 731         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 732         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 733         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello world\n"), Equals, true)
 734
 735 }
 736
 737 func (s *TestSuite) TestCrunchstat(c *C) {
 738         api, _, _ := FullRunHelper(c, `{
 739                 "command": ["sleep", "1"],
 740                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 741                 "cwd": ".",
 742                 "environment": {},
 743                 "mounts": {"/tmp": {"kind": "tmp"} },
 744                 "output_path": "/tmp",
 745                 "priority": 1,
 746                 "runtime_constraints": {}
 747         }`, nil, 0, func(t *TestDockerClient) {
 748                 time.Sleep(time.Second)
 749                 t.logWriter.Close()
 750         })
 751
 752         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 753         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 754
 755         // We didn't actually start a container, so crunchstat didn't
 756         // find accounting files and therefore didn't log any stats.
 757         // It should have logged a "can't find accounting files"
 758         // message after one poll interval, though, so we can confirm
 759         // it's alive:
 760         c.Assert(api.Logs["crunchstat"], NotNil)
 761         c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 762
 763         // The "files never appeared" log assures us that we called
 764         // (*crunchstat.Reporter)Stop(), and that we set it up with
 765         // the correct container ID "abcde":
 766         c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for abcde\n`)
 767 }
 768
 769 func (s *TestSuite) TestNodeInfoLog(c *C) {
 770         api, _, _ := FullRunHelper(c, `{
 771                 "command": ["sleep", "1"],
 772                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 773                 "cwd": ".",
 774                 "environment": {},
 775                 "mounts": {"/tmp": {"kind": "tmp"} },
 776                 "output_path": "/tmp",
 777                 "priority": 1,
 778                 "runtime_constraints": {}
 779         }`, nil, 0,
 780                 func(t *TestDockerClient) {
 781                         time.Sleep(time.Second)
 782                         t.logWriter.Close()
 783                 })
 784
 785         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 786         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 787
 788         c.Assert(api.Logs["node-info"], NotNil)
 789         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Host Information.*`)
 790         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`)
 791         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`)
 792         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`)
 793         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk INodes.*`)
 794 }
 795
 796 func (s *TestSuite) TestContainerRecordLog(c *C) {
 797         api, _, _ := FullRunHelper(c, `{
 798                 "command": ["sleep", "1"],
 799                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 800                 "cwd": ".",
 801                 "environment": {},
 802                 "mounts": {"/tmp": {"kind": "tmp"} },
 803                 "output_path": "/tmp",
 804                 "priority": 1,
 805                 "runtime_constraints": {}
 806         }`, nil, 0,
 807                 func(t *TestDockerClient) {
 808                         time.Sleep(time.Second)
 809                         t.logWriter.Close()
 810                 })
 811
 812         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 813         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 814
 815         c.Assert(api.Logs["container"], NotNil)
 816         c.Check(api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 817 }
 818
 819 func (s *TestSuite) TestFullRunStderr(c *C) {
 820         api, _, _ := FullRunHelper(c, `{
 821     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 822     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 823     "cwd": ".",
 824     "environment": {},
 825     "mounts": {"/tmp": {"kind": "tmp"} },
 826     "output_path": "/tmp",
 827     "priority": 1,
 828     "runtime_constraints": {}
 829 }`, nil, 1, func(t *TestDockerClient) {
 830                 t.logWriter.Write(dockerLog(1, "hello\n"))
 831                 t.logWriter.Write(dockerLog(2, "world\n"))
 832                 t.logWriter.Close()
 833         })
 834
 835         final := api.CalledWith("container.state", "Complete")
 836         c.Assert(final, NotNil)
 837         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 838         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 839
 840         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello\n"), Equals, true)
 841         c.Check(strings.HasSuffix(api.Logs["stderr"].String(), "world\n"), Equals, true)
 842 }
 843
 844 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 845         api, _, _ := FullRunHelper(c, `{
 846     "command": ["pwd"],
 847     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 848     "cwd": ".",
 849     "environment": {},
 850     "mounts": {"/tmp": {"kind": "tmp"} },
 851     "output_path": "/tmp",
 852     "priority": 1,
 853     "runtime_constraints": {}
 854 }`, nil, 0, func(t *TestDockerClient) {
 855                 t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 856                 t.logWriter.Close()
 857         })
 858
 859         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 860         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 861         c.Log(api.Logs["stdout"])
 862         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/\n"), Equals, true)
 863 }
 864
 865 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 866         api, _, _ := FullRunHelper(c, `{
 867     "command": ["pwd"],
 868     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 869     "cwd": "/bin",
 870     "environment": {},
 871     "mounts": {"/tmp": {"kind": "tmp"} },
 872     "output_path": "/tmp",
 873     "priority": 1,
 874     "runtime_constraints": {}
 875 }`, nil, 0, func(t *TestDockerClient) {
 876                 t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 877                 t.logWriter.Close()
 878         })
 879
 880         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 881         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 882         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/bin\n"), Equals, true)
 883 }
 884
 885 func (s *TestSuite) TestStopOnSignal(c *C) {
 886         s.testStopContainer(c, func(cr *ContainerRunner) {
 887                 go func() {
 888                         for !cr.cStarted {
 889                                 time.Sleep(time.Millisecond)
 890                         }
 891                         cr.SigChan <- syscall.SIGINT
 892                 }()
 893         })
 894 }
 895
 896 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 897         s.testStopContainer(c, func(cr *ContainerRunner) {
 898                 cr.ArvMountExit = make(chan error)
 899                 go func() {
 900                         cr.ArvMountExit <- exec.Command("true").Run()
 901                         close(cr.ArvMountExit)
 902                 }()
 903         })
 904 }
 905
 906 func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
 907         record := `{
 908     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 909     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 910     "cwd": ".",
 911     "environment": {},
 912     "mounts": {"/tmp": {"kind": "tmp"} },
 913     "output_path": "/tmp",
 914     "priority": 1,
 915     "runtime_constraints": {}
 916 }`
 917
 918         rec := arvados.Container{}
 919         err := json.Unmarshal([]byte(record), &rec)
 920         c.Check(err, IsNil)
 921
 922         docker := NewTestDockerClient(0)
 923         docker.fn = func(t *TestDockerClient) {
 924                 <-t.stop
 925                 t.logWriter.Write(dockerLog(1, "foo\n"))
 926                 t.logWriter.Close()
 927         }
 928         docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 929
 930         api := &ArvTestClient{Container: rec}
 931         cr := NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 932         cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
 933         setup(cr)
 934
 935         done := make(chan error)
 936         go func() {
 937                 done <- cr.Run()
 938         }()
 939         select {
 940         case <-time.After(20 * time.Second):
 941                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
 942                 c.Fatal("timed out")
 943         case err = <-done:
 944                 c.Check(err, IsNil)
 945         }
 946         for k, v := range api.Logs {
 947                 c.Log(k)
 948                 c.Log(v.String())
 949         }
 950
 951         c.Check(api.CalledWith("container.log", nil), NotNil)
 952         c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
 953         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "foo\n"), Equals, true)
 954 }
 955
 956 func (s *TestSuite) TestFullRunSetEnv(c *C) {
 957         api, _, _ := FullRunHelper(c, `{
 958     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
 959     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 960     "cwd": "/bin",
 961     "environment": {"FROBIZ": "bilbo"},
 962     "mounts": {"/tmp": {"kind": "tmp"} },
 963     "output_path": "/tmp",
 964     "priority": 1,
 965     "runtime_constraints": {}
 966 }`, nil, 0, func(t *TestDockerClient) {
 967                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 968                 t.logWriter.Close()
 969         })
 970
 971         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 972         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 973         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "bilbo\n"), Equals, true)
 974 }
 975
 976 type ArvMountCmdLine struct {
 977         Cmd   []string
 978         token string
 979 }
 980
 981 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
 982         am.Cmd = c
 983         am.token = token
 984         return nil, nil
 985 }
 986
 987 func stubCert(temp string) string {
 988         path := temp + "/ca-certificates.crt"
 989         crt, _ := os.Create(path)
 990         crt.Close()
 991         arvadosclient.CertFiles = []string{path}
 992         return path
 993 }
 994
 995 func (s *TestSuite) TestSetupMounts(c *C) {
 996         api := &ArvTestClient{}
 997         kc := &KeepTestClient{}
 998         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 999         am := &ArvMountCmdLine{}
1000         cr.RunArvMount = am.ArvMountTest
1001
1002         realTemp, err := ioutil.TempDir("", "crunchrun_test1-")
1003         c.Assert(err, IsNil)
1004         certTemp, err := ioutil.TempDir("", "crunchrun_test2-")
1005         c.Assert(err, IsNil)
1006         stubCertPath := stubCert(certTemp)
1007
1008         defer os.RemoveAll(realTemp)
1009         defer os.RemoveAll(certTemp)
1010
1011         i := 0
1012         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1013                 i++
1014                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1015                 err := os.Mkdir(d, os.ModePerm)
1016                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1017                         // Test case must have pre-populated the tempdir
1018                         err = nil
1019                 }
1020                 return d, err
1021         }
1022
1023         checkEmpty := func() {
1024                 filepath.Walk(realTemp, func(path string, _ os.FileInfo, err error) error {
1025                         c.Check(path, Equals, realTemp)
1026                         c.Check(err, IsNil)
1027                         return nil
1028                 })
1029         }
1030
1031         {
1032                 i = 0
1033                 cr.ArvMountPoint = ""
1034                 cr.Container.Mounts = make(map[string]arvados.Mount)
1035                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1036                 cr.OutputPath = "/tmp"
1037                 cr.statInterval = 5 * time.Second
1038                 err := cr.SetupMounts()
1039                 c.Check(err, IsNil)
1040                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1041                         "--read-write", "--crunchstat-interval=5",
1042                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1043                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp"})
1044                 os.RemoveAll(cr.ArvMountPoint)
1045                 cr.CleanupDirs()
1046                 checkEmpty()
1047         }
1048
1049         {
1050                 i = 0
1051                 cr.ArvMountPoint = ""
1052                 cr.Container.Mounts = make(map[string]arvados.Mount)
1053                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1054                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1055                 cr.OutputPath = "/out"
1056
1057                 err := cr.SetupMounts()
1058                 c.Check(err, IsNil)
1059                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1060                         "--read-write", "--crunchstat-interval=5",
1061                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1062                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/out", realTemp + "/3:/tmp"})
1063                 os.RemoveAll(cr.ArvMountPoint)
1064                 cr.CleanupDirs()
1065                 checkEmpty()
1066         }
1067
1068         {
1069                 i = 0
1070                 cr.ArvMountPoint = ""
1071                 cr.Container.Mounts = make(map[string]arvados.Mount)
1072                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1073                 cr.OutputPath = "/tmp"
1074
1075                 apiflag := true
1076                 cr.Container.RuntimeConstraints.API = &apiflag
1077
1078                 err := cr.SetupMounts()
1079                 c.Check(err, IsNil)
1080                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1081                         "--read-write", "--crunchstat-interval=5",
1082                         "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1083                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", stubCertPath + ":/etc/arvados/ca-certificates.crt:ro"})
1084                 os.RemoveAll(cr.ArvMountPoint)
1085                 cr.CleanupDirs()
1086                 checkEmpty()
1087
1088                 apiflag = false
1089         }
1090
1091         {
1092                 i = 0
1093                 cr.ArvMountPoint = ""
1094                 cr.Container.Mounts = map[string]arvados.Mount{
1095                         "/keeptmp": {Kind: "collection", Writable: true},
1096                 }
1097                 cr.OutputPath = "/keeptmp"
1098
1099                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1100
1101                 err := cr.SetupMounts()
1102                 c.Check(err, IsNil)
1103                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1104                         "--read-write", "--crunchstat-interval=5",
1105                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1106                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/tmp0:/keeptmp"})
1107                 os.RemoveAll(cr.ArvMountPoint)
1108                 cr.CleanupDirs()
1109                 checkEmpty()
1110         }
1111
1112         {
1113                 i = 0
1114                 cr.ArvMountPoint = ""
1115                 cr.Container.Mounts = map[string]arvados.Mount{
1116                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1117                         "/keepout": {Kind: "collection", Writable: true},
1118                 }
1119                 cr.OutputPath = "/keepout"
1120
1121                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1122                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1123
1124                 err := cr.SetupMounts()
1125                 c.Check(err, IsNil)
1126                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1127                         "--read-write", "--crunchstat-interval=5",
1128                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1129                 sort.StringSlice(cr.Binds).Sort()
1130                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
1131                         realTemp + "/keep1/tmp0:/keepout"})
1132                 os.RemoveAll(cr.ArvMountPoint)
1133                 cr.CleanupDirs()
1134                 checkEmpty()
1135         }
1136
1137         {
1138                 i = 0
1139                 cr.ArvMountPoint = ""
1140                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1141                 cr.Container.Mounts = map[string]arvados.Mount{
1142                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1143                         "/keepout": {Kind: "collection", Writable: true},
1144                 }
1145                 cr.OutputPath = "/keepout"
1146
1147                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1148                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1149
1150                 err := cr.SetupMounts()
1151                 c.Check(err, IsNil)
1152                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1153                         "--read-write", "--crunchstat-interval=5",
1154                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1155                 sort.StringSlice(cr.Binds).Sort()
1156                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
1157                         realTemp + "/keep1/tmp0:/keepout"})
1158                 os.RemoveAll(cr.ArvMountPoint)
1159                 cr.CleanupDirs()
1160                 checkEmpty()
1161         }
1162
1163         for _, test := range []struct {
1164                 in  interface{}
1165                 out string
1166         }{
1167                 {in: "foo", out: `"foo"`},
1168                 {in: nil, out: `null`},
1169                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1170         } {
1171                 i = 0
1172                 cr.ArvMountPoint = ""
1173                 cr.Container.Mounts = map[string]arvados.Mount{
1174                         "/mnt/test.json": {Kind: "json", Content: test.in},
1175                 }
1176                 err := cr.SetupMounts()
1177                 c.Check(err, IsNil)
1178                 sort.StringSlice(cr.Binds).Sort()
1179                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2/mountdata.json:/mnt/test.json:ro"})
1180                 content, err := ioutil.ReadFile(realTemp + "/2/mountdata.json")
1181                 c.Check(err, IsNil)
1182                 c.Check(content, DeepEquals, []byte(test.out))
1183                 os.RemoveAll(cr.ArvMountPoint)
1184                 cr.CleanupDirs()
1185                 checkEmpty()
1186         }
1187
1188         // Read-only mount points are allowed underneath output_dir mount point
1189         {
1190                 i = 0
1191                 cr.ArvMountPoint = ""
1192                 cr.Container.Mounts = make(map[string]arvados.Mount)
1193                 cr.Container.Mounts = map[string]arvados.Mount{
1194                         "/tmp":     {Kind: "tmp"},
1195                         "/tmp/foo": {Kind: "collection"},
1196                 }
1197                 cr.OutputPath = "/tmp"
1198
1199                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1200
1201                 err := cr.SetupMounts()
1202                 c.Check(err, IsNil)
1203                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other",
1204                         "--read-write", "--crunchstat-interval=5",
1205                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1206                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", realTemp + "/keep1/tmp0:/tmp/foo:ro"})
1207                 os.RemoveAll(cr.ArvMountPoint)
1208                 cr.CleanupDirs()
1209                 checkEmpty()
1210         }
1211
1212         // Writable mount points are not allowed underneath output_dir mount point
1213         {
1214                 i = 0
1215                 cr.ArvMountPoint = ""
1216                 cr.Container.Mounts = make(map[string]arvados.Mount)
1217                 cr.Container.Mounts = map[string]arvados.Mount{
1218                         "/tmp":     {Kind: "tmp"},
1219                         "/tmp/foo": {Kind: "collection", Writable: true},
1220                 }
1221                 cr.OutputPath = "/tmp"
1222
1223                 err := cr.SetupMounts()
1224                 c.Check(err, NotNil)
1225                 c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`)
1226                 os.RemoveAll(cr.ArvMountPoint)
1227                 cr.CleanupDirs()
1228                 checkEmpty()
1229         }
1230
1231         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1232         {
1233                 i = 0
1234                 cr.ArvMountPoint = ""
1235                 cr.Container.Mounts = make(map[string]arvados.Mount)
1236                 cr.Container.Mounts = map[string]arvados.Mount{
1237                         "/tmp":     {Kind: "tmp"},
1238                         "/tmp/foo": {Kind: "json"},
1239                 }
1240                 cr.OutputPath = "/tmp"
1241
1242                 err := cr.SetupMounts()
1243                 c.Check(err, NotNil)
1244                 c.Check(err, ErrorMatches, `Only mount points of kind 'collection' are supported underneath the output_path.*`)
1245                 os.RemoveAll(cr.ArvMountPoint)
1246                 cr.CleanupDirs()
1247                 checkEmpty()
1248         }
1249
1250         // Only mount point of kind 'collection' is allowed for stdin
1251         {
1252                 i = 0
1253                 cr.ArvMountPoint = ""
1254                 cr.Container.Mounts = make(map[string]arvados.Mount)
1255                 cr.Container.Mounts = map[string]arvados.Mount{
1256                         "stdin": {Kind: "tmp"},
1257                 }
1258
1259                 err := cr.SetupMounts()
1260                 c.Check(err, NotNil)
1261                 c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`)
1262                 os.RemoveAll(cr.ArvMountPoint)
1263                 cr.CleanupDirs()
1264                 checkEmpty()
1265         }
1266 }
1267
1268 func (s *TestSuite) TestStdout(c *C) {
1269         helperRecord := `{
1270                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1271                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1272                 "cwd": "/bin",
1273                 "environment": {"FROBIZ": "bilbo"},
1274                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1275                 "output_path": "/tmp",
1276                 "priority": 1,
1277                 "runtime_constraints": {}
1278         }`
1279
1280         api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
1281                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1282                 t.logWriter.Close()
1283         })
1284
1285         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1286         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1287         c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1288 }
1289
1290 // Used by the TestStdoutWithWrongPath*()
1291 func StdoutErrorRunHelper(c *C, record string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, err error) {
1292         rec := arvados.Container{}
1293         err = json.Unmarshal([]byte(record), &rec)
1294         c.Check(err, IsNil)
1295
1296         docker := NewTestDockerClient(0)
1297         docker.fn = fn
1298         docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
1299
1300         api = &ArvTestClient{Container: rec}
1301         cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
1302         am := &ArvMountCmdLine{}
1303         cr.RunArvMount = am.ArvMountTest
1304
1305         err = cr.Run()
1306         return
1307 }
1308
1309 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1310         _, _, err := StdoutErrorRunHelper(c, `{
1311     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1312     "output_path": "/tmp"
1313 }`, func(t *TestDockerClient) {})
1314
1315         c.Check(err, NotNil)
1316         c.Check(strings.Contains(err.Error(), "Stdout path does not start with OutputPath"), Equals, true)
1317 }
1318
1319 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1320         _, _, err := StdoutErrorRunHelper(c, `{
1321     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1322     "output_path": "/tmp"
1323 }`, func(t *TestDockerClient) {})
1324
1325         c.Check(err, NotNil)
1326         c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'tmp' for stdout"), Equals, true)
1327 }
1328
1329 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1330         _, _, err := StdoutErrorRunHelper(c, `{
1331     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1332     "output_path": "/tmp"
1333 }`, func(t *TestDockerClient) {})
1334
1335         c.Check(err, NotNil)
1336         c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'collection' for stdout"), Equals, true)
1337 }
1338
1339 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1340         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1341         defer os.Unsetenv("ARVADOS_API_HOST")
1342         api, _, _ := FullRunHelper(c, `{
1343     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1344     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1345     "cwd": "/bin",
1346     "environment": {},
1347     "mounts": {"/tmp": {"kind": "tmp"} },
1348     "output_path": "/tmp",
1349     "priority": 1,
1350     "runtime_constraints": {"API": true}
1351 }`, nil, 0, func(t *TestDockerClient) {
1352                 t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
1353                 t.logWriter.Close()
1354         })
1355
1356         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1357         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1358         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "test.arvados.org\n"), Equals, true)
1359         c.Check(api.CalledWith("container.output", "d41d8cd98f00b204e9800998ecf8427e+0"), NotNil)
1360 }
1361
1362 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1363         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1364         defer os.Unsetenv("ARVADOS_API_HOST")
1365         api, _, _ := FullRunHelper(c, `{
1366     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1367     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1368     "cwd": "/bin",
1369     "environment": {},
1370     "mounts": {"/tmp": {"kind": "tmp"} },
1371     "output_path": "/tmp",
1372     "priority": 1,
1373     "runtime_constraints": {"API": true}
1374 }`, nil, 0, func(t *TestDockerClient) {
1375                 t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
1376                 t.logWriter.Close()
1377         })
1378
1379         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1380         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1381         c.Check(api.CalledWith("container.output", "d4ab34d3d4f8a72f5c4973051ae69fab+122"), NotNil)
1382 }
1383
1384 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1385         helperRecord := `{
1386                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1387                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1388                 "cwd": "/bin",
1389                 "environment": {"FROBIZ": "bilbo"},
1390                 "mounts": {
1391         "/tmp": {"kind": "tmp"},
1392         "/tmp/foo": {"kind": "collection",
1393                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1394                      "exclude_from_output": true
1395         },
1396         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1397     },
1398                 "output_path": "/tmp",
1399                 "priority": 1,
1400                 "runtime_constraints": {}
1401         }`
1402
1403         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1404
1405         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1406                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1407                 t.logWriter.Close()
1408         })
1409
1410         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1411         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1412         c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1413 }
1414
1415 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1416         helperRecord := `{
1417                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1418                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1419                 "cwd": "/bin",
1420                 "environment": {"FROBIZ": "bilbo"},
1421                 "mounts": {
1422         "/tmp": {"kind": "tmp"},
1423         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1424         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1425         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1426         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1427         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1428     },
1429                 "output_path": "/tmp",
1430                 "priority": 1,
1431                 "runtime_constraints": {}
1432         }`
1433
1434         extraMounts := []string{
1435                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1436                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1437                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1438         }
1439
1440         api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1441                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1442                 t.logWriter.Close()
1443         })
1444
1445         c.Check(runner.Binds, DeepEquals, []string{realtemp + "/2:/tmp",
1446                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt:/tmp/foo/bar:ro",
1447                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt:/tmp/foo/baz/sub2file2:ro",
1448                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1:/tmp/foo/sub1:ro",
1449                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt:/tmp/foo/sub1file2:ro",
1450         })
1451
1452         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1453         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1454         for _, v := range api.Content {
1455                 if v["collection"] != nil {
1456                         c.Check(v["ensure_unique_name"], Equals, true)
1457                         collection := v["collection"].(arvadosclient.Dict)
1458                         if strings.Index(collection["name"].(string), "output") == 0 {
1459                                 manifest := collection["manifest_text"].(string)
1460
1461                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1462 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 9:18:bar 9:18:sub1file2
1463 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1464 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1465 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1466 `)
1467                         }
1468                 }
1469         }
1470 }
1471
1472 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1473         helperRecord := `{
1474                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1475                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1476                 "cwd": "/bin",
1477                 "environment": {"FROBIZ": "bilbo"},
1478                 "mounts": {
1479         "/tmp": {"kind": "tmp"},
1480         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
1481         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1482     },
1483                 "output_path": "/tmp",
1484                 "priority": 1,
1485                 "runtime_constraints": {}
1486         }`
1487
1488         extraMounts := []string{
1489                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1490         }
1491
1492         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1493                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1494                 t.logWriter.Close()
1495         })
1496
1497         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1498         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1499         for _, v := range api.Content {
1500                 if v["collection"] != nil {
1501                         collection := v["collection"].(arvadosclient.Dict)
1502                         if strings.Index(collection["name"].(string), "output") == 0 {
1503                                 manifest := collection["manifest_text"].(string)
1504
1505                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1506 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1507 `)
1508                         }
1509                 }
1510         }
1511 }
1512
1513 func (s *TestSuite) TestOutputSymlinkToInput(c *C) {
1514         helperRecord := `{
1515                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1516                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1517                 "cwd": "/bin",
1518                 "environment": {"FROBIZ": "bilbo"},
1519                 "mounts": {
1520         "/tmp": {"kind": "tmp"},
1521         "/keep/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1522         "/keep/foo2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367"}
1523     },
1524                 "output_path": "/tmp",
1525                 "priority": 1,
1526                 "runtime_constraints": {}
1527         }`
1528
1529         extraMounts := []string{
1530                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1531         }
1532
1533         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1534                 os.Symlink("/keep/foo/sub1file2", t.realTemp+"/2/baz")
1535                 os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz2")
1536                 os.Symlink("/keep/foo2/subdir1", t.realTemp+"/2/baz3")
1537                 os.Mkdir(t.realTemp+"/2/baz4", 0700)
1538                 os.Symlink("/keep/foo2/subdir1/file2_in_subdir1.txt", t.realTemp+"/2/baz4/baz5")
1539                 t.logWriter.Close()
1540         })
1541
1542         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1543         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1544         for _, v := range api.Content {
1545                 if v["collection"] != nil {
1546                         collection := v["collection"].(arvadosclient.Dict)
1547                         if strings.Index(collection["name"].(string), "output") == 0 {
1548                                 manifest := collection["manifest_text"].(string)
1549                                 c.Check(manifest, Equals, `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:baz 9:18:baz2
1550 ./baz3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1551 ./baz3/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1552 ./baz4 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:baz5
1553 `)
1554                         }
1555                 }
1556         }
1557 }
1558
1559 func (s *TestSuite) TestOutputError(c *C) {
1560         helperRecord := `{
1561                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1562                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1563                 "cwd": "/bin",
1564                 "environment": {"FROBIZ": "bilbo"},
1565                 "mounts": {
1566         "/tmp": {"kind": "tmp"}
1567     },
1568                 "output_path": "/tmp",
1569                 "priority": 1,
1570                 "runtime_constraints": {}
1571         }`
1572
1573         extraMounts := []string{}
1574
1575         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1576                 os.Symlink("/etc/hosts", t.realTemp+"/2/baz")
1577                 t.logWriter.Close()
1578         })
1579
1580         c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
1581 }
1582
1583 func (s *TestSuite) TestOutputSymlinkToOutput(c *C) {
1584         helperRecord := `{
1585                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1586                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1587                 "cwd": "/bin",
1588                 "environment": {"FROBIZ": "bilbo"},
1589                 "mounts": {
1590         "/tmp": {"kind": "tmp"}
1591     },
1592                 "output_path": "/tmp",
1593                 "priority": 1,
1594                 "runtime_constraints": {}
1595         }`
1596
1597         extraMounts := []string{}
1598
1599         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1600                 rf, _ := os.Create(t.realTemp + "/2/realfile")
1601                 rf.Write([]byte("foo"))
1602                 rf.Close()
1603
1604                 os.Mkdir(t.realTemp+"/2/realdir", 0700)
1605                 rf, _ = os.Create(t.realTemp + "/2/realdir/subfile")
1606                 rf.Write([]byte("bar"))
1607                 rf.Close()
1608
1609                 os.Symlink("/tmp/realfile", t.realTemp+"/2/file1")
1610                 os.Symlink("realfile", t.realTemp+"/2/file2")
1611                 os.Symlink("/tmp/file1", t.realTemp+"/2/file3")
1612                 os.Symlink("file2", t.realTemp+"/2/file4")
1613                 os.Symlink("realdir", t.realTemp+"/2/dir1")
1614                 os.Symlink("/tmp/realdir", t.realTemp+"/2/dir2")
1615                 t.logWriter.Close()
1616         })
1617
1618         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1619         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1620         for _, v := range api.Content {
1621                 if v["collection"] != nil {
1622                         collection := v["collection"].(arvadosclient.Dict)
1623                         if strings.Index(collection["name"].(string), "output") == 0 {
1624                                 manifest := collection["manifest_text"].(string)
1625                                 c.Check(manifest, Equals,
1626                                         `. 7a2c86e102dcc231bd232aad99686dfa+15 0:3:file1 3:3:file2 6:3:file3 9:3:file4 12:3:realfile
1627 ./dir1 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
1628 ./dir2 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
1629 ./realdir 37b51d194a7513e45b56f6524f2d51f2+3 0:3:subfile
1630 `)
1631                         }
1632                 }
1633         }
1634 }
1635
1636 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1637         helperRecord := `{
1638                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1639                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1640                 "cwd": "/bin",
1641                 "environment": {"FROBIZ": "bilbo"},
1642                 "mounts": {
1643         "/tmp": {"kind": "tmp"},
1644         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1645         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1646     },
1647                 "output_path": "/tmp",
1648                 "priority": 1,
1649                 "runtime_constraints": {}
1650         }`
1651
1652         extraMounts := []string{
1653                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1654         }
1655
1656         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1657                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1658                 t.logWriter.Close()
1659         })
1660
1661         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1662         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1663         for _, v := range api.Content {
1664                 if v["collection"] != nil {
1665                         collection := v["collection"].(arvadosclient.Dict)
1666                         if strings.Index(collection["name"].(string), "output") == 0 {
1667                                 manifest := collection["manifest_text"].(string)
1668                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1669 `)
1670                         }
1671                 }
1672         }
1673 }
1674
1675 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1676         helperRecord := `{
1677                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1678                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1679                 "cwd": "/bin",
1680                 "environment": {"FROBIZ": "bilbo"},
1681                 "mounts": {
1682         "/tmp": {"kind": "tmp"},
1683         "stdin": {"kind": "json", "content": "foo"},
1684         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1685     },
1686                 "output_path": "/tmp",
1687                 "priority": 1,
1688                 "runtime_constraints": {}
1689         }`
1690
1691         api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
1692                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1693                 t.logWriter.Close()
1694         })
1695
1696         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1697         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1698         for _, v := range api.Content {
1699                 if v["collection"] != nil {
1700                         collection := v["collection"].(arvadosclient.Dict)
1701                         if strings.Index(collection["name"].(string), "output") == 0 {
1702                                 manifest := collection["manifest_text"].(string)
1703                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1704 `)
1705                         }
1706                 }
1707         }
1708 }
1709
1710 func (s *TestSuite) TestStderrMount(c *C) {
1711         api, _, _ := FullRunHelper(c, `{
1712     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1713     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1714     "cwd": ".",
1715     "environment": {},
1716     "mounts": {"/tmp": {"kind": "tmp"},
1717                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1718                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1719     "output_path": "/tmp",
1720     "priority": 1,
1721     "runtime_constraints": {}
1722 }`, nil, 1, func(t *TestDockerClient) {
1723                 t.logWriter.Write(dockerLog(1, "hello\n"))
1724                 t.logWriter.Write(dockerLog(2, "oops\n"))
1725                 t.logWriter.Close()
1726         })
1727
1728         final := api.CalledWith("container.state", "Complete")
1729         c.Assert(final, NotNil)
1730         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1731         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1732
1733         c.Check(api.CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1734 }
1735
1736 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1737         cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
1738         cr.fetchContainerRecord()
1739
1740         jsondata, err := json.Marshal(cr.Container.Mounts["/json"].Content)
1741
1742         c.Check(err, IsNil)
1743         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1744 }
1745
1746 func (s *TestSuite) TestEvalSymlinks(c *C) {
1747         cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
1748
1749         realTemp, err := ioutil.TempDir("", "crunchrun_test-")
1750         c.Assert(err, IsNil)
1751         defer os.RemoveAll(realTemp)
1752
1753         cr.HostOutputDir = realTemp
1754
1755         // Absolute path outside output dir
1756         os.Symlink("/etc/passwd", realTemp+"/p1")
1757
1758         // Relative outside output dir
1759         os.Symlink("../zip", realTemp+"/p2")
1760
1761         // Circular references
1762         os.Symlink("p4", realTemp+"/p3")
1763         os.Symlink("p5", realTemp+"/p4")
1764         os.Symlink("p3", realTemp+"/p5")
1765
1766         // Target doesn't exist
1767         os.Symlink("p99", realTemp+"/p6")
1768
1769         for _, v := range []string{"p1", "p2", "p3", "p4", "p5"} {
1770                 info, err := os.Lstat(realTemp + "/" + v)
1771                 _, _, _, err = cr.derefOutputSymlink(realTemp+"/"+v, info)
1772                 c.Assert(err, NotNil)
1773         }
1774 }
1775
1776 func (s *TestSuite) TestEvalSymlinkDir(c *C) {
1777         cr := NewContainerRunner(&ArvTestClient{callraw: true}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
1778
1779         realTemp, err := ioutil.TempDir("", "crunchrun_test-")
1780         c.Assert(err, IsNil)
1781         defer os.RemoveAll(realTemp)
1782
1783         cr.HostOutputDir = realTemp
1784
1785         // Absolute path outside output dir
1786         os.Symlink(".", realTemp+"/loop")
1787
1788         v := "loop"
1789         info, err := os.Lstat(realTemp + "/" + v)
1790         _, err = cr.UploadOutputFile(realTemp+"/"+v, info, err, []string{}, nil, "", "", 0)
1791         c.Assert(err, NotNil)
1792 }
1793
1794 func (s *TestSuite) TestFullBrokenDocker1(c *C) {
1795         tf, err := ioutil.TempFile("", "brokenNodeHook-")
1796         c.Assert(err, IsNil)
1797         defer os.Remove(tf.Name())
1798
1799         tf.Write([]byte(`#!/bin/sh
1800 exec echo killme
1801 `))
1802         tf.Close()
1803         os.Chmod(tf.Name(), 0700)
1804
1805         ech := tf.Name()
1806         brokenNodeHook = &ech
1807
1808         api, _, _ := FullRunHelper(c, `{
1809     "command": ["echo", "hello world"],
1810     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1811     "cwd": ".",
1812     "environment": {},
1813     "mounts": {"/tmp": {"kind": "tmp"} },
1814     "output_path": "/tmp",
1815     "priority": 1,
1816     "runtime_constraints": {}
1817 }`, nil, 2, func(t *TestDockerClient) {
1818                 t.logWriter.Write(dockerLog(1, "hello world\n"))
1819                 t.logWriter.Close()
1820         })
1821
1822         c.Check(api.CalledWith("container.state", "Queued"), NotNil)
1823         c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1824         c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1825         c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1826
1827 }
1828
1829 func (s *TestSuite) TestFullBrokenDocker2(c *C) {
1830         ech := ""
1831         brokenNodeHook = &ech
1832
1833         api, _, _ := FullRunHelper(c, `{
1834     "command": ["echo", "hello world"],
1835     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1836     "cwd": ".",
1837     "environment": {},
1838     "mounts": {"/tmp": {"kind": "tmp"} },
1839     "output_path": "/tmp",
1840     "priority": 1,
1841     "runtime_constraints": {}
1842 }`, nil, 2, func(t *TestDockerClient) {
1843                 t.logWriter.Write(dockerLog(1, "hello world\n"))
1844                 t.logWriter.Close()
1845         })
1846
1847         c.Check(api.CalledWith("container.state", "Queued"), NotNil)
1848         c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1849         c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*No broken node hook.*")
1850 }
1851
1852 func (s *TestSuite) TestFullBrokenDocker3(c *C) {
1853         ech := ""
1854         brokenNodeHook = &ech
1855
1856         api, _, _ := FullRunHelper(c, `{
1857     "command": ["echo", "hello world"],
1858     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1859     "cwd": ".",
1860     "environment": {},
1861     "mounts": {"/tmp": {"kind": "tmp"} },
1862     "output_path": "/tmp",
1863     "priority": 1,
1864     "runtime_constraints": {}
1865 }`, nil, 3, func(t *TestDockerClient) {
1866                 t.logWriter.Write(dockerLog(1, "hello world\n"))
1867                 t.logWriter.Close()
1868         })
1869
1870         c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
1871         c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1872 }
1873
1874 func (s *TestSuite) TestBadCommand1(c *C) {
1875         ech := ""
1876         brokenNodeHook = &ech
1877
1878         api, _, _ := FullRunHelper(c, `{
1879     "command": ["echo", "hello world"],
1880     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1881     "cwd": ".",
1882     "environment": {},
1883     "mounts": {"/tmp": {"kind": "tmp"} },
1884     "output_path": "/tmp",
1885     "priority": 1,
1886     "runtime_constraints": {}
1887 }`, nil, 4, func(t *TestDockerClient) {
1888                 t.logWriter.Write(dockerLog(1, "hello world\n"))
1889                 t.logWriter.Close()
1890         })
1891
1892         c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
1893         c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1894 }
1895
1896 func (s *TestSuite) TestBadCommand2(c *C) {
1897         ech := ""
1898         brokenNodeHook = &ech
1899
1900         api, _, _ := FullRunHelper(c, `{
1901     "command": ["echo", "hello world"],
1902     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1903     "cwd": ".",
1904     "environment": {},
1905     "mounts": {"/tmp": {"kind": "tmp"} },
1906     "output_path": "/tmp",
1907     "priority": 1,
1908     "runtime_constraints": {}
1909 }`, nil, 5, func(t *TestDockerClient) {
1910                 t.logWriter.Write(dockerLog(1, "hello world\n"))
1911                 t.logWriter.Close()
1912         })
1913
1914         c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
1915         c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1916 }
1917
1918 func (s *TestSuite) TestBadCommand3(c *C) {
1919         ech := ""
1920         brokenNodeHook = &ech
1921
1922         api, _, _ := FullRunHelper(c, `{
1923     "command": ["echo", "hello world"],
1924     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1925     "cwd": ".",
1926     "environment": {},
1927     "mounts": {"/tmp": {"kind": "tmp"} },
1928     "output_path": "/tmp",
1929     "priority": 1,
1930     "runtime_constraints": {}
1931 }`, nil, 6, func(t *TestDockerClient) {
1932                 t.logWriter.Write(dockerLog(1, "hello world\n"))
1933                 t.logWriter.Close()
1934         })
1935
1936         c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
1937         c.Check(api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1938 }