services/crunch-run/crunchrun_test.go

   1 package main
   2
   3 import (
   4         "bufio"
   5         "bytes"
   6         "context"
   7         "crypto/md5"
   8         "encoding/json"
   9         "errors"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net"
  14         "os"
  15         "os/exec"
  16         "path/filepath"
  17         "runtime/pprof"
  18         "sort"
  19         "strings"
  20         "sync"
  21         "syscall"
  22         "testing"
  23         "time"
  24
  25         "git.curoverse.com/arvados.git/sdk/go/arvados"
  26         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  27         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  28         "git.curoverse.com/arvados.git/sdk/go/manifest"
  29
  30         dockertypes "github.com/docker/docker/api/types"
  31         dockercontainer "github.com/docker/docker/api/types/container"
  32         dockernetwork "github.com/docker/docker/api/types/network"
  33         . "gopkg.in/check.v1"
  34 )
  35
  36 // Gocheck boilerplate
  37 func TestCrunchExec(t *testing.T) {
  38         TestingT(t)
  39 }
  40
  41 type TestSuite struct{}
  42
  43 // Gocheck boilerplate
  44 var _ = Suite(&TestSuite{})
  45
  46 type ArvTestClient struct {
  47         Total   int64
  48         Calls   int
  49         Content []arvadosclient.Dict
  50         arvados.Container
  51         Logs map[string]*bytes.Buffer
  52         sync.Mutex
  53         WasSetRunning bool
  54 }
  55
  56 type KeepTestClient struct {
  57         Called  bool
  58         Content []byte
  59 }
  60
  61 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
  62 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
  63 var hwImageId = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
  64
  65 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
  66 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
  67
  68 var normalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt\n./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt\n./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt\n"
  69 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
  70
  71 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
  72 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
  73
  74 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
  75 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
  76
  77 type TestDockerClient struct {
  78         imageLoaded string
  79         logReader   io.ReadCloser
  80         logWriter   io.WriteCloser
  81         fn          func(t *TestDockerClient)
  82         finish      int
  83         stop        chan bool
  84         cwd         string
  85         env         []string
  86         api         *ArvTestClient
  87 }
  88
  89 func NewTestDockerClient(exitCode int) *TestDockerClient {
  90         t := &TestDockerClient{}
  91         t.logReader, t.logWriter = io.Pipe()
  92         t.finish = exitCode
  93         t.stop = make(chan bool)
  94         t.cwd = "/"
  95         return t
  96 }
  97
  98 type MockConn struct {
  99         net.Conn
 100 }
 101
 102 func (m *MockConn) Write(b []byte) (int, error) {
 103         return len(b), nil
 104 }
 105
 106 func (m *MockConn) Close() error {
 107         return nil
 108 }
 109
 110 func NewMockConn() *MockConn {
 111         c := &MockConn{}
 112         return c
 113 }
 114
 115 func (t *TestDockerClient) ContainerAttach(ctx context.Context, container string, options dockertypes.ContainerAttachOptions) (dockertypes.HijackedResponse, error) {
 116         return dockertypes.HijackedResponse{Conn: NewMockConn(), Reader: bufio.NewReader(t.logReader)}, nil
 117 }
 118
 119 func (t *TestDockerClient) ContainerCreate(ctx context.Context, config *dockercontainer.Config, hostConfig *dockercontainer.HostConfig, networkingConfig *dockernetwork.NetworkingConfig, containerName string) (dockercontainer.ContainerCreateCreatedBody, error) {
 120         if config.WorkingDir != "" {
 121                 t.cwd = config.WorkingDir
 122         }
 123         t.env = config.Env
 124         return dockercontainer.ContainerCreateCreatedBody{ID: "abcde"}, nil
 125 }
 126
 127 func (t *TestDockerClient) ContainerStart(ctx context.Context, container string, options dockertypes.ContainerStartOptions) error {
 128         if container == "abcde" {
 129                 go t.fn(t)
 130                 return nil
 131         } else {
 132                 return errors.New("Invalid container id")
 133         }
 134 }
 135
 136 func (t *TestDockerClient) ContainerStop(ctx context.Context, container string, timeout *time.Duration) error {
 137         t.stop <- true
 138         return nil
 139 }
 140
 141 func (t *TestDockerClient) ContainerWait(ctx context.Context, container string) (int64, error) {
 142         return int64(t.finish), nil
 143 }
 144
 145 func (t *TestDockerClient) ImageInspectWithRaw(ctx context.Context, image string) (dockertypes.ImageInspect, []byte, error) {
 146         if t.imageLoaded == image {
 147                 return dockertypes.ImageInspect{}, nil, nil
 148         } else {
 149                 return dockertypes.ImageInspect{}, nil, errors.New("")
 150         }
 151 }
 152
 153 func (t *TestDockerClient) ImageLoad(ctx context.Context, input io.Reader, quiet bool) (dockertypes.ImageLoadResponse, error) {
 154         _, err := io.Copy(ioutil.Discard, input)
 155         if err != nil {
 156                 return dockertypes.ImageLoadResponse{}, err
 157         } else {
 158                 t.imageLoaded = hwImageId
 159                 return dockertypes.ImageLoadResponse{Body: ioutil.NopCloser(input)}, nil
 160         }
 161 }
 162
 163 func (*TestDockerClient) ImageRemove(ctx context.Context, image string, options dockertypes.ImageRemoveOptions) ([]dockertypes.ImageDeleteResponseItem, error) {
 164         return nil, nil
 165 }
 166
 167 func (client *ArvTestClient) Create(resourceType string,
 168         parameters arvadosclient.Dict,
 169         output interface{}) error {
 170
 171         client.Mutex.Lock()
 172         defer client.Mutex.Unlock()
 173
 174         client.Calls++
 175         client.Content = append(client.Content, parameters)
 176
 177         if resourceType == "logs" {
 178                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
 179                 if client.Logs == nil {
 180                         client.Logs = make(map[string]*bytes.Buffer)
 181                 }
 182                 if client.Logs[et] == nil {
 183                         client.Logs[et] = &bytes.Buffer{}
 184                 }
 185                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
 186         }
 187
 188         if resourceType == "collections" && output != nil {
 189                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
 190                 outmap := output.(*arvados.Collection)
 191                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
 192         }
 193
 194         return nil
 195 }
 196
 197 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 198         switch {
 199         case method == "GET" && resourceType == "containers" && action == "auth":
 200                 return json.Unmarshal([]byte(`{
 201                         "kind": "arvados#api_client_authorization",
 202                         "uuid": "`+fakeAuthUUID+`",
 203                         "api_token": "`+fakeAuthToken+`"
 204                         }`), output)
 205         default:
 206                 return fmt.Errorf("Not found")
 207         }
 208 }
 209
 210 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
 211         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 212         j := []byte(`{
 213                 "command": ["sleep", "1"],
 214                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 215                 "cwd": ".",
 216                 "environment": {},
 217                 "mounts": {"/tmp": {"kind": "tmp"} },
 218                 "output_path": "/tmp",
 219                 "priority": 1,
 220                 "runtime_constraints": {}
 221         }`)
 222         return ioutil.NopCloser(bytes.NewReader(j)), nil
 223 }
 224
 225 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 226         if resourceType == "collections" {
 227                 if uuid == hwPDH {
 228                         output.(*arvados.Collection).ManifestText = hwManifest
 229                 } else if uuid == otherPDH {
 230                         output.(*arvados.Collection).ManifestText = otherManifest
 231                 } else if uuid == normalizedWithSubdirsPDH {
 232                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
 233                 } else if uuid == denormalizedWithSubdirsPDH {
 234                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
 235                 }
 236         }
 237         if resourceType == "containers" {
 238                 (*output.(*arvados.Container)) = client.Container
 239         }
 240         return nil
 241 }
 242
 243 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 244         client.Mutex.Lock()
 245         defer client.Mutex.Unlock()
 246         client.Calls++
 247         client.Content = append(client.Content, parameters)
 248         if resourceType == "containers" {
 249                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
 250                         client.WasSetRunning = true
 251                 }
 252         }
 253         return nil
 254 }
 255
 256 var discoveryMap = map[string]interface{}{"defaultTrashLifetime": float64(1209600)}
 257
 258 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
 259         return discoveryMap[key], nil
 260 }
 261
 262 // CalledWith returns the parameters from the first API call whose
 263 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
 264 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
 265 // no call matches, it returns nil.
 266 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
 267 call:
 268         for _, content := range client.Content {
 269                 var v interface{} = content
 270                 for _, k := range strings.Split(jpath, ".") {
 271                         if dict, ok := v.(arvadosclient.Dict); !ok {
 272                                 continue call
 273                         } else {
 274                                 v = dict[k]
 275                         }
 276                 }
 277                 if v == expect {
 278                         return content
 279                 }
 280         }
 281         return nil
 282 }
 283
 284 func (client *KeepTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 285         client.Content = buf
 286         return fmt.Sprintf("%s+%d", hash, len(buf)), len(buf), nil
 287 }
 288
 289 type FileWrapper struct {
 290         io.ReadCloser
 291         len uint64
 292 }
 293
 294 func (fw FileWrapper) Len() uint64 {
 295         return fw.len
 296 }
 297
 298 func (fw FileWrapper) Seek(int64, int) (int64, error) {
 299         return 0, errors.New("not implemented")
 300 }
 301
 302 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
 303         if filename == hwImageId+".tar" {
 304                 rdr := ioutil.NopCloser(&bytes.Buffer{})
 305                 client.Called = true
 306                 return FileWrapper{rdr, 1321984}, nil
 307         } else if filename == "/file1_in_main.txt" {
 308                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
 309                 client.Called = true
 310                 return FileWrapper{rdr, 3}, nil
 311         }
 312         return nil, nil
 313 }
 314
 315 func (s *TestSuite) TestLoadImage(c *C) {
 316         kc := &KeepTestClient{}
 317         docker := NewTestDockerClient(0)
 318         cr := NewContainerRunner(&ArvTestClient{}, kc, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 319
 320         _, err := cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 321
 322         _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
 323         c.Check(err, NotNil)
 324
 325         cr.Container.ContainerImage = hwPDH
 326
 327         // (1) Test loading image from keep
 328         c.Check(kc.Called, Equals, false)
 329         c.Check(cr.ContainerConfig.Image, Equals, "")
 330
 331         err = cr.LoadImage()
 332
 333         c.Check(err, IsNil)
 334         defer func() {
 335                 cr.Docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 336         }()
 337
 338         c.Check(kc.Called, Equals, true)
 339         c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
 340
 341         _, _, err = cr.Docker.ImageInspectWithRaw(nil, hwImageId)
 342         c.Check(err, IsNil)
 343
 344         // (2) Test using image that's already loaded
 345         kc.Called = false
 346         cr.ContainerConfig.Image = ""
 347
 348         err = cr.LoadImage()
 349         c.Check(err, IsNil)
 350         c.Check(kc.Called, Equals, false)
 351         c.Check(cr.ContainerConfig.Image, Equals, hwImageId)
 352
 353 }
 354
 355 type ArvErrorTestClient struct{}
 356
 357 func (ArvErrorTestClient) Create(resourceType string,
 358         parameters arvadosclient.Dict,
 359         output interface{}) error {
 360         return nil
 361 }
 362
 363 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
 364         return errors.New("ArvError")
 365 }
 366
 367 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
 368         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
 369         return nil, errors.New("ArvError")
 370 }
 371
 372 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
 373         return errors.New("ArvError")
 374 }
 375
 376 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
 377         return nil
 378 }
 379
 380 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
 381         return discoveryMap[key], nil
 382 }
 383
 384 type KeepErrorTestClient struct{}
 385
 386 func (KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 387         return "", 0, errors.New("KeepError")
 388 }
 389
 390 func (KeepErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
 391         return nil, errors.New("KeepError")
 392 }
 393
 394 type KeepReadErrorTestClient struct{}
 395
 396 func (KeepReadErrorTestClient) PutHB(hash string, buf []byte) (string, int, error) {
 397         return "", 0, nil
 398 }
 399
 400 type ErrorReader struct{}
 401
 402 func (ErrorReader) Read(p []byte) (n int, err error) {
 403         return 0, errors.New("ErrorReader")
 404 }
 405
 406 func (ErrorReader) Close() error {
 407         return nil
 408 }
 409
 410 func (ErrorReader) Len() uint64 {
 411         return 0
 412 }
 413
 414 func (ErrorReader) Seek(int64, int) (int64, error) {
 415         return 0, errors.New("ErrorReader")
 416 }
 417
 418 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (keepclient.Reader, error) {
 419         return ErrorReader{}, nil
 420 }
 421
 422 func (s *TestSuite) TestLoadImageArvError(c *C) {
 423         // (1) Arvados error
 424         cr := NewContainerRunner(ArvErrorTestClient{}, &KeepTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 425         cr.Container.ContainerImage = hwPDH
 426
 427         err := cr.LoadImage()
 428         c.Check(err.Error(), Equals, "While getting container image collection: ArvError")
 429 }
 430
 431 func (s *TestSuite) TestLoadImageKeepError(c *C) {
 432         // (2) Keep error
 433         docker := NewTestDockerClient(0)
 434         cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 435         cr.Container.ContainerImage = hwPDH
 436
 437         err := cr.LoadImage()
 438         c.Check(err.Error(), Equals, "While creating ManifestFileReader for container image: KeepError")
 439 }
 440
 441 func (s *TestSuite) TestLoadImageCollectionError(c *C) {
 442         // (3) Collection doesn't contain image
 443         cr := NewContainerRunner(&ArvTestClient{}, KeepErrorTestClient{}, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 444         cr.Container.ContainerImage = otherPDH
 445
 446         err := cr.LoadImage()
 447         c.Check(err.Error(), Equals, "First file in the container image collection does not end in .tar")
 448 }
 449
 450 func (s *TestSuite) TestLoadImageKeepReadError(c *C) {
 451         // (4) Collection doesn't contain image
 452         docker := NewTestDockerClient(0)
 453         cr := NewContainerRunner(&ArvTestClient{}, KeepReadErrorTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 454         cr.Container.ContainerImage = hwPDH
 455
 456         err := cr.LoadImage()
 457         c.Check(err, NotNil)
 458 }
 459
 460 type ClosableBuffer struct {
 461         bytes.Buffer
 462 }
 463
 464 func (*ClosableBuffer) Close() error {
 465         return nil
 466 }
 467
 468 type TestLogs struct {
 469         Stdout ClosableBuffer
 470         Stderr ClosableBuffer
 471 }
 472
 473 func (tl *TestLogs) NewTestLoggingWriter(logstr string) io.WriteCloser {
 474         if logstr == "stdout" {
 475                 return &tl.Stdout
 476         }
 477         if logstr == "stderr" {
 478                 return &tl.Stderr
 479         }
 480         return nil
 481 }
 482
 483 func dockerLog(fd byte, msg string) []byte {
 484         by := []byte(msg)
 485         header := make([]byte, 8+len(by))
 486         header[0] = fd
 487         header[7] = byte(len(by))
 488         copy(header[8:], by)
 489         return header
 490 }
 491
 492 func (s *TestSuite) TestRunContainer(c *C) {
 493         docker := NewTestDockerClient(0)
 494         docker.fn = func(t *TestDockerClient) {
 495                 t.logWriter.Write(dockerLog(1, "Hello world\n"))
 496                 t.logWriter.Close()
 497         }
 498         cr := NewContainerRunner(&ArvTestClient{}, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 499
 500         var logs TestLogs
 501         cr.NewLogWriter = logs.NewTestLoggingWriter
 502         cr.Container.ContainerImage = hwPDH
 503         cr.Container.Command = []string{"./hw"}
 504         err := cr.LoadImage()
 505         c.Check(err, IsNil)
 506
 507         err = cr.CreateContainer()
 508         c.Check(err, IsNil)
 509
 510         err = cr.StartContainer()
 511         c.Check(err, IsNil)
 512
 513         err = cr.WaitFinish()
 514         c.Check(err, IsNil)
 515
 516         c.Check(strings.HasSuffix(logs.Stdout.String(), "Hello world\n"), Equals, true)
 517         c.Check(logs.Stderr.String(), Equals, "")
 518 }
 519
 520 func (s *TestSuite) TestCommitLogs(c *C) {
 521         api := &ArvTestClient{}
 522         kc := &KeepTestClient{}
 523         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 524         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
 525
 526         cr.CrunchLog.Print("Hello world!")
 527         cr.CrunchLog.Print("Goodbye")
 528         cr.finalState = "Complete"
 529
 530         err := cr.CommitLogs()
 531         c.Check(err, IsNil)
 532
 533         c.Check(api.Calls, Equals, 2)
 534         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
 535         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 536         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
 537         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
 538 }
 539
 540 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
 541         api := &ArvTestClient{}
 542         kc := &KeepTestClient{}
 543         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 544
 545         err := cr.UpdateContainerRunning()
 546         c.Check(err, IsNil)
 547
 548         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
 549 }
 550
 551 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
 552         api := &ArvTestClient{}
 553         kc := &KeepTestClient{}
 554         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 555
 556         cr.LogsPDH = new(string)
 557         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
 558
 559         cr.ExitCode = new(int)
 560         *cr.ExitCode = 42
 561         cr.finalState = "Complete"
 562
 563         err := cr.UpdateContainerFinal()
 564         c.Check(err, IsNil)
 565
 566         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
 567         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
 568         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
 569 }
 570
 571 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
 572         api := &ArvTestClient{}
 573         kc := &KeepTestClient{}
 574         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 575         cr.cCancelled = true
 576         cr.finalState = "Cancelled"
 577
 578         err := cr.UpdateContainerFinal()
 579         c.Check(err, IsNil)
 580
 581         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
 582         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
 583         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
 584 }
 585
 586 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
 587 // dress rehearsal of the Run() function, starting from a JSON container record.
 588 func FullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, realTemp string) {
 589         rec := arvados.Container{}
 590         err := json.Unmarshal([]byte(record), &rec)
 591         c.Check(err, IsNil)
 592
 593         docker := NewTestDockerClient(exitCode)
 594         docker.fn = fn
 595         docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 596
 597         api = &ArvTestClient{Container: rec}
 598         docker.api = api
 599         cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 600         cr.statInterval = 100 * time.Millisecond
 601         am := &ArvMountCmdLine{}
 602         cr.RunArvMount = am.ArvMountTest
 603
 604         realTemp, err = ioutil.TempDir("", "crunchrun_test1-")
 605         c.Assert(err, IsNil)
 606         defer os.RemoveAll(realTemp)
 607
 608         tempcount := 0
 609         cr.MkTempDir = func(_ string, prefix string) (string, error) {
 610                 tempcount++
 611                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
 612                 err := os.Mkdir(d, os.ModePerm)
 613                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 614                         // Test case must have pre-populated the tempdir
 615                         err = nil
 616                 }
 617                 return d, err
 618         }
 619
 620         if extraMounts != nil && len(extraMounts) > 0 {
 621                 err := cr.SetupArvMountPoint("keep")
 622                 c.Check(err, IsNil)
 623
 624                 for _, m := range extraMounts {
 625                         os.MkdirAll(cr.ArvMountPoint+"/by_id/"+m, os.ModePerm)
 626                 }
 627         }
 628
 629         err = cr.Run()
 630         c.Check(err, IsNil)
 631         c.Check(api.WasSetRunning, Equals, true)
 632
 633         c.Check(api.Content[api.Calls-1]["container"].(arvadosclient.Dict)["log"], NotNil)
 634
 635         if err != nil {
 636                 for k, v := range api.Logs {
 637                         c.Log(k)
 638                         c.Log(v.String())
 639                 }
 640         }
 641
 642         return
 643 }
 644
 645 func (s *TestSuite) TestFullRunHello(c *C) {
 646         api, _, _ := FullRunHelper(c, `{
 647     "command": ["echo", "hello world"],
 648     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 649     "cwd": ".",
 650     "environment": {},
 651     "mounts": {"/tmp": {"kind": "tmp"} },
 652     "output_path": "/tmp",
 653     "priority": 1,
 654     "runtime_constraints": {}
 655 }`, nil, 0, func(t *TestDockerClient) {
 656                 t.logWriter.Write(dockerLog(1, "hello world\n"))
 657                 t.logWriter.Close()
 658         })
 659
 660         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 661         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 662         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello world\n"), Equals, true)
 663
 664 }
 665
 666 func (s *TestSuite) TestCrunchstat(c *C) {
 667         api, _, _ := FullRunHelper(c, `{
 668                 "command": ["sleep", "1"],
 669                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 670                 "cwd": ".",
 671                 "environment": {},
 672                 "mounts": {"/tmp": {"kind": "tmp"} },
 673                 "output_path": "/tmp",
 674                 "priority": 1,
 675                 "runtime_constraints": {}
 676         }`, nil, 0, func(t *TestDockerClient) {
 677                 time.Sleep(time.Second)
 678                 t.logWriter.Close()
 679         })
 680
 681         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 682         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 683
 684         // We didn't actually start a container, so crunchstat didn't
 685         // find accounting files and therefore didn't log any stats.
 686         // It should have logged a "can't find accounting files"
 687         // message after one poll interval, though, so we can confirm
 688         // it's alive:
 689         c.Assert(api.Logs["crunchstat"], NotNil)
 690         c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
 691
 692         // The "files never appeared" log assures us that we called
 693         // (*crunchstat.Reporter)Stop(), and that we set it up with
 694         // the correct container ID "abcde":
 695         c.Check(api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for abcde\n`)
 696 }
 697
 698 func (s *TestSuite) TestNodeInfoLog(c *C) {
 699         api, _, _ := FullRunHelper(c, `{
 700                 "command": ["sleep", "1"],
 701                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 702                 "cwd": ".",
 703                 "environment": {},
 704                 "mounts": {"/tmp": {"kind": "tmp"} },
 705                 "output_path": "/tmp",
 706                 "priority": 1,
 707                 "runtime_constraints": {}
 708         }`, nil, 0,
 709                 func(t *TestDockerClient) {
 710                         time.Sleep(time.Second)
 711                         t.logWriter.Close()
 712                 })
 713
 714         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 715         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 716
 717         c.Assert(api.Logs["node-info"], NotNil)
 718         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Host Information.*`)
 719         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*CPU Information.*`)
 720         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Memory Information.*`)
 721         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk Space.*`)
 722         c.Check(api.Logs["node-info"].String(), Matches, `(?ms).*Disk INodes.*`)
 723 }
 724
 725 func (s *TestSuite) TestContainerRecordLog(c *C) {
 726         api, _, _ := FullRunHelper(c, `{
 727                 "command": ["sleep", "1"],
 728                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 729                 "cwd": ".",
 730                 "environment": {},
 731                 "mounts": {"/tmp": {"kind": "tmp"} },
 732                 "output_path": "/tmp",
 733                 "priority": 1,
 734                 "runtime_constraints": {}
 735         }`, nil, 0,
 736                 func(t *TestDockerClient) {
 737                         time.Sleep(time.Second)
 738                         t.logWriter.Close()
 739                 })
 740
 741         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 742         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 743
 744         c.Assert(api.Logs["container"], NotNil)
 745         c.Check(api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
 746 }
 747
 748 func (s *TestSuite) TestFullRunStderr(c *C) {
 749         api, _, _ := FullRunHelper(c, `{
 750     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
 751     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 752     "cwd": ".",
 753     "environment": {},
 754     "mounts": {"/tmp": {"kind": "tmp"} },
 755     "output_path": "/tmp",
 756     "priority": 1,
 757     "runtime_constraints": {}
 758 }`, nil, 1, func(t *TestDockerClient) {
 759                 t.logWriter.Write(dockerLog(1, "hello\n"))
 760                 t.logWriter.Write(dockerLog(2, "world\n"))
 761                 t.logWriter.Close()
 762         })
 763
 764         final := api.CalledWith("container.state", "Complete")
 765         c.Assert(final, NotNil)
 766         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
 767         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
 768
 769         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "hello\n"), Equals, true)
 770         c.Check(strings.HasSuffix(api.Logs["stderr"].String(), "world\n"), Equals, true)
 771 }
 772
 773 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
 774         api, _, _ := FullRunHelper(c, `{
 775     "command": ["pwd"],
 776     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 777     "cwd": ".",
 778     "environment": {},
 779     "mounts": {"/tmp": {"kind": "tmp"} },
 780     "output_path": "/tmp",
 781     "priority": 1,
 782     "runtime_constraints": {}
 783 }`, nil, 0, func(t *TestDockerClient) {
 784                 t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 785                 t.logWriter.Close()
 786         })
 787
 788         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 789         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 790         c.Log(api.Logs["stdout"])
 791         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/\n"), Equals, true)
 792 }
 793
 794 func (s *TestSuite) TestFullRunSetCwd(c *C) {
 795         api, _, _ := FullRunHelper(c, `{
 796     "command": ["pwd"],
 797     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 798     "cwd": "/bin",
 799     "environment": {},
 800     "mounts": {"/tmp": {"kind": "tmp"} },
 801     "output_path": "/tmp",
 802     "priority": 1,
 803     "runtime_constraints": {}
 804 }`, nil, 0, func(t *TestDockerClient) {
 805                 t.logWriter.Write(dockerLog(1, t.cwd+"\n"))
 806                 t.logWriter.Close()
 807         })
 808
 809         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 810         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 811         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "/bin\n"), Equals, true)
 812 }
 813
 814 func (s *TestSuite) TestStopOnSignal(c *C) {
 815         s.testStopContainer(c, func(cr *ContainerRunner) {
 816                 go func() {
 817                         for !cr.cStarted {
 818                                 time.Sleep(time.Millisecond)
 819                         }
 820                         cr.SigChan <- syscall.SIGINT
 821                 }()
 822         })
 823 }
 824
 825 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
 826         s.testStopContainer(c, func(cr *ContainerRunner) {
 827                 cr.ArvMountExit = make(chan error)
 828                 go func() {
 829                         cr.ArvMountExit <- exec.Command("true").Run()
 830                         close(cr.ArvMountExit)
 831                 }()
 832         })
 833 }
 834
 835 func (s *TestSuite) testStopContainer(c *C, setup func(cr *ContainerRunner)) {
 836         record := `{
 837     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
 838     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 839     "cwd": ".",
 840     "environment": {},
 841     "mounts": {"/tmp": {"kind": "tmp"} },
 842     "output_path": "/tmp",
 843     "priority": 1,
 844     "runtime_constraints": {}
 845 }`
 846
 847         rec := arvados.Container{}
 848         err := json.Unmarshal([]byte(record), &rec)
 849         c.Check(err, IsNil)
 850
 851         docker := NewTestDockerClient(0)
 852         docker.fn = func(t *TestDockerClient) {
 853                 <-t.stop
 854                 t.logWriter.Write(dockerLog(1, "foo\n"))
 855                 t.logWriter.Close()
 856         }
 857         docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
 858
 859         api := &ArvTestClient{Container: rec}
 860         cr := NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 861         cr.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
 862         setup(cr)
 863
 864         done := make(chan error)
 865         go func() {
 866                 done <- cr.Run()
 867         }()
 868         select {
 869         case <-time.After(20 * time.Second):
 870                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
 871                 c.Fatal("timed out")
 872         case err = <-done:
 873                 c.Check(err, IsNil)
 874         }
 875         for k, v := range api.Logs {
 876                 c.Log(k)
 877                 c.Log(v.String())
 878         }
 879
 880         c.Check(api.CalledWith("container.log", nil), NotNil)
 881         c.Check(api.CalledWith("container.state", "Cancelled"), NotNil)
 882         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "foo\n"), Equals, true)
 883 }
 884
 885 func (s *TestSuite) TestFullRunSetEnv(c *C) {
 886         api, _, _ := FullRunHelper(c, `{
 887     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
 888     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
 889     "cwd": "/bin",
 890     "environment": {"FROBIZ": "bilbo"},
 891     "mounts": {"/tmp": {"kind": "tmp"} },
 892     "output_path": "/tmp",
 893     "priority": 1,
 894     "runtime_constraints": {}
 895 }`, nil, 0, func(t *TestDockerClient) {
 896                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
 897                 t.logWriter.Close()
 898         })
 899
 900         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
 901         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
 902         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "bilbo\n"), Equals, true)
 903 }
 904
 905 type ArvMountCmdLine struct {
 906         Cmd   []string
 907         token string
 908 }
 909
 910 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
 911         am.Cmd = c
 912         am.token = token
 913         return nil, nil
 914 }
 915
 916 func stubCert(temp string) string {
 917         path := temp + "/ca-certificates.crt"
 918         crt, _ := os.Create(path)
 919         crt.Close()
 920         arvadosclient.CertFiles = []string{path}
 921         return path
 922 }
 923
 924 func (s *TestSuite) TestSetupMounts(c *C) {
 925         api := &ArvTestClient{}
 926         kc := &KeepTestClient{}
 927         cr := NewContainerRunner(api, kc, nil, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
 928         am := &ArvMountCmdLine{}
 929         cr.RunArvMount = am.ArvMountTest
 930
 931         realTemp, err := ioutil.TempDir("", "crunchrun_test1-")
 932         c.Assert(err, IsNil)
 933         certTemp, err := ioutil.TempDir("", "crunchrun_test2-")
 934         c.Assert(err, IsNil)
 935         stubCertPath := stubCert(certTemp)
 936
 937         defer os.RemoveAll(realTemp)
 938         defer os.RemoveAll(certTemp)
 939
 940         i := 0
 941         cr.MkTempDir = func(_ string, prefix string) (string, error) {
 942                 i++
 943                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
 944                 err := os.Mkdir(d, os.ModePerm)
 945                 if err != nil && strings.Contains(err.Error(), ": file exists") {
 946                         // Test case must have pre-populated the tempdir
 947                         err = nil
 948                 }
 949                 return d, err
 950         }
 951
 952         checkEmpty := func() {
 953                 filepath.Walk(realTemp, func(path string, _ os.FileInfo, err error) error {
 954                         c.Check(path, Equals, realTemp)
 955                         c.Check(err, IsNil)
 956                         return nil
 957                 })
 958         }
 959
 960         {
 961                 i = 0
 962                 cr.ArvMountPoint = ""
 963                 cr.Container.Mounts = make(map[string]arvados.Mount)
 964                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
 965                 cr.OutputPath = "/tmp"
 966
 967                 err := cr.SetupMounts()
 968                 c.Check(err, IsNil)
 969                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 970                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp"})
 971                 cr.CleanupDirs()
 972                 checkEmpty()
 973         }
 974
 975         {
 976                 i = 0
 977                 cr.ArvMountPoint = ""
 978                 cr.Container.Mounts = make(map[string]arvados.Mount)
 979                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
 980                 cr.OutputPath = "/tmp"
 981
 982                 apiflag := true
 983                 cr.Container.RuntimeConstraints.API = &apiflag
 984
 985                 err := cr.SetupMounts()
 986                 c.Check(err, IsNil)
 987                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
 988                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", stubCertPath + ":/etc/arvados/ca-certificates.crt:ro"})
 989                 cr.CleanupDirs()
 990                 checkEmpty()
 991
 992                 apiflag = false
 993         }
 994
 995         {
 996                 i = 0
 997                 cr.ArvMountPoint = ""
 998                 cr.Container.Mounts = map[string]arvados.Mount{
 999                         "/keeptmp": {Kind: "collection", Writable: true},
1000                 }
1001                 cr.OutputPath = "/keeptmp"
1002
1003                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1004
1005                 err := cr.SetupMounts()
1006                 c.Check(err, IsNil)
1007                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1008                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/tmp0:/keeptmp"})
1009                 cr.CleanupDirs()
1010                 checkEmpty()
1011         }
1012
1013         {
1014                 i = 0
1015                 cr.ArvMountPoint = ""
1016                 cr.Container.Mounts = map[string]arvados.Mount{
1017                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1018                         "/keepout": {Kind: "collection", Writable: true},
1019                 }
1020                 cr.OutputPath = "/keepout"
1021
1022                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1023                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1024
1025                 err := cr.SetupMounts()
1026                 c.Check(err, IsNil)
1027                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1028                 sort.StringSlice(cr.Binds).Sort()
1029                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
1030                         realTemp + "/keep1/tmp0:/keepout"})
1031                 cr.CleanupDirs()
1032                 checkEmpty()
1033         }
1034
1035         {
1036                 i = 0
1037                 cr.ArvMountPoint = ""
1038                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1039                 cr.Container.Mounts = map[string]arvados.Mount{
1040                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1041                         "/keepout": {Kind: "collection", Writable: true},
1042                 }
1043                 cr.OutputPath = "/keepout"
1044
1045                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1046                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1047
1048                 err := cr.SetupMounts()
1049                 c.Check(err, IsNil)
1050                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1051                 sort.StringSlice(cr.Binds).Sort()
1052                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53:/keepinp:ro",
1053                         realTemp + "/keep1/tmp0:/keepout"})
1054                 cr.CleanupDirs()
1055                 checkEmpty()
1056         }
1057
1058         for _, test := range []struct {
1059                 in  interface{}
1060                 out string
1061         }{
1062                 {in: "foo", out: `"foo"`},
1063                 {in: nil, out: `null`},
1064                 {in: map[string]int{"foo": 123}, out: `{"foo":123}`},
1065         } {
1066                 i = 0
1067                 cr.ArvMountPoint = ""
1068                 cr.Container.Mounts = map[string]arvados.Mount{
1069                         "/mnt/test.json": {Kind: "json", Content: test.in},
1070                 }
1071                 err := cr.SetupMounts()
1072                 c.Check(err, IsNil)
1073                 sort.StringSlice(cr.Binds).Sort()
1074                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2/mountdata.json:/mnt/test.json:ro"})
1075                 content, err := ioutil.ReadFile(realTemp + "/2/mountdata.json")
1076                 c.Check(err, IsNil)
1077                 c.Check(content, DeepEquals, []byte(test.out))
1078                 cr.CleanupDirs()
1079                 checkEmpty()
1080         }
1081
1082         // Read-only mount points are allowed underneath output_dir mount point
1083         {
1084                 i = 0
1085                 cr.ArvMountPoint = ""
1086                 cr.Container.Mounts = make(map[string]arvados.Mount)
1087                 cr.Container.Mounts = map[string]arvados.Mount{
1088                         "/tmp":     {Kind: "tmp"},
1089                         "/tmp/foo": {Kind: "collection"},
1090                 }
1091                 cr.OutputPath = "/tmp"
1092
1093                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1094
1095                 err := cr.SetupMounts()
1096                 c.Check(err, IsNil)
1097                 c.Check(am.Cmd, DeepEquals, []string{"--foreground", "--allow-other", "--read-write", "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", realTemp + "/keep1"})
1098                 c.Check(cr.Binds, DeepEquals, []string{realTemp + "/2:/tmp", realTemp + "/keep1/tmp0:/tmp/foo:ro"})
1099                 cr.CleanupDirs()
1100                 checkEmpty()
1101         }
1102
1103         // Writable mount points are not allowed underneath output_dir mount point
1104         {
1105                 i = 0
1106                 cr.ArvMountPoint = ""
1107                 cr.Container.Mounts = make(map[string]arvados.Mount)
1108                 cr.Container.Mounts = map[string]arvados.Mount{
1109                         "/tmp":     {Kind: "tmp"},
1110                         "/tmp/foo": {Kind: "collection", Writable: true},
1111                 }
1112                 cr.OutputPath = "/tmp"
1113
1114                 err := cr.SetupMounts()
1115                 c.Check(err, NotNil)
1116                 c.Check(err, ErrorMatches, `Writable mount points are not permitted underneath the output_path.*`)
1117                 cr.CleanupDirs()
1118                 checkEmpty()
1119         }
1120
1121         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1122         {
1123                 i = 0
1124                 cr.ArvMountPoint = ""
1125                 cr.Container.Mounts = make(map[string]arvados.Mount)
1126                 cr.Container.Mounts = map[string]arvados.Mount{
1127                         "/tmp":     {Kind: "tmp"},
1128                         "/tmp/foo": {Kind: "json"},
1129                 }
1130                 cr.OutputPath = "/tmp"
1131
1132                 err := cr.SetupMounts()
1133                 c.Check(err, NotNil)
1134                 c.Check(err, ErrorMatches, `Only mount points of kind 'collection' are supported underneath the output_path.*`)
1135                 cr.CleanupDirs()
1136                 checkEmpty()
1137         }
1138
1139         // Only mount point of kind 'collection' is allowed for stdin
1140         {
1141                 i = 0
1142                 cr.ArvMountPoint = ""
1143                 cr.Container.Mounts = make(map[string]arvados.Mount)
1144                 cr.Container.Mounts = map[string]arvados.Mount{
1145                         "stdin": {Kind: "tmp"},
1146                 }
1147
1148                 err := cr.SetupMounts()
1149                 c.Check(err, NotNil)
1150                 c.Check(err, ErrorMatches, `Unsupported mount kind 'tmp' for stdin.*`)
1151                 cr.CleanupDirs()
1152                 checkEmpty()
1153         }
1154 }
1155
1156 func (s *TestSuite) TestStdout(c *C) {
1157         helperRecord := `{
1158                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1159                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1160                 "cwd": "/bin",
1161                 "environment": {"FROBIZ": "bilbo"},
1162                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1163                 "output_path": "/tmp",
1164                 "priority": 1,
1165                 "runtime_constraints": {}
1166         }`
1167
1168         api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
1169                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1170                 t.logWriter.Close()
1171         })
1172
1173         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1174         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1175         c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1176 }
1177
1178 // Used by the TestStdoutWithWrongPath*()
1179 func StdoutErrorRunHelper(c *C, record string, fn func(t *TestDockerClient)) (api *ArvTestClient, cr *ContainerRunner, err error) {
1180         rec := arvados.Container{}
1181         err = json.Unmarshal([]byte(record), &rec)
1182         c.Check(err, IsNil)
1183
1184         docker := NewTestDockerClient(0)
1185         docker.fn = fn
1186         docker.ImageRemove(nil, hwImageId, dockertypes.ImageRemoveOptions{})
1187
1188         api = &ArvTestClient{Container: rec}
1189         cr = NewContainerRunner(api, &KeepTestClient{}, docker, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
1190         am := &ArvMountCmdLine{}
1191         cr.RunArvMount = am.ArvMountTest
1192
1193         err = cr.Run()
1194         return
1195 }
1196
1197 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1198         _, _, err := StdoutErrorRunHelper(c, `{
1199     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1200     "output_path": "/tmp"
1201 }`, func(t *TestDockerClient) {})
1202
1203         c.Check(err, NotNil)
1204         c.Check(strings.Contains(err.Error(), "Stdout path does not start with OutputPath"), Equals, true)
1205 }
1206
1207 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1208         _, _, err := StdoutErrorRunHelper(c, `{
1209     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1210     "output_path": "/tmp"
1211 }`, func(t *TestDockerClient) {})
1212
1213         c.Check(err, NotNil)
1214         c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'tmp' for stdout"), Equals, true)
1215 }
1216
1217 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1218         _, _, err := StdoutErrorRunHelper(c, `{
1219     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1220     "output_path": "/tmp"
1221 }`, func(t *TestDockerClient) {})
1222
1223         c.Check(err, NotNil)
1224         c.Check(strings.Contains(err.Error(), "Unsupported mount kind 'collection' for stdout"), Equals, true)
1225 }
1226
1227 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1228         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1229         defer os.Unsetenv("ARVADOS_API_HOST")
1230         api, _, _ := FullRunHelper(c, `{
1231     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1232     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1233     "cwd": "/bin",
1234     "environment": {},
1235     "mounts": {"/tmp": {"kind": "tmp"} },
1236     "output_path": "/tmp",
1237     "priority": 1,
1238     "runtime_constraints": {"API": true}
1239 }`, nil, 0, func(t *TestDockerClient) {
1240                 t.logWriter.Write(dockerLog(1, t.env[1][17:]+"\n"))
1241                 t.logWriter.Close()
1242         })
1243
1244         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1245         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1246         c.Check(strings.HasSuffix(api.Logs["stdout"].String(), "test.arvados.org\n"), Equals, true)
1247         c.Check(api.CalledWith("container.output", "d41d8cd98f00b204e9800998ecf8427e+0"), NotNil)
1248 }
1249
1250 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1251         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1252         defer os.Unsetenv("ARVADOS_API_HOST")
1253         api, _, _ := FullRunHelper(c, `{
1254     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1255     "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1256     "cwd": "/bin",
1257     "environment": {},
1258     "mounts": {"/tmp": {"kind": "tmp"} },
1259     "output_path": "/tmp",
1260     "priority": 1,
1261     "runtime_constraints": {"API": true}
1262 }`, nil, 0, func(t *TestDockerClient) {
1263                 t.api.Container.Output = "d4ab34d3d4f8a72f5c4973051ae69fab+122"
1264                 t.logWriter.Close()
1265         })
1266
1267         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1268         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1269         c.Check(api.CalledWith("container.output", "d4ab34d3d4f8a72f5c4973051ae69fab+122"), NotNil)
1270 }
1271
1272 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1273         helperRecord := `{
1274                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1275                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1276                 "cwd": "/bin",
1277                 "environment": {"FROBIZ": "bilbo"},
1278                 "mounts": {
1279         "/tmp": {"kind": "tmp"},
1280         "/tmp/foo": {"kind": "collection",
1281                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1282                      "exclude_from_output": true
1283         },
1284         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1285     },
1286                 "output_path": "/tmp",
1287                 "priority": 1,
1288                 "runtime_constraints": {}
1289         }`
1290
1291         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1292
1293         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1294                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1295                 t.logWriter.Close()
1296         })
1297
1298         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1299         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1300         c.Check(api.CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1301 }
1302
1303 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1304         helperRecord := `{
1305                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1306                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1307                 "cwd": "/bin",
1308                 "environment": {"FROBIZ": "bilbo"},
1309                 "mounts": {
1310         "/tmp": {"kind": "tmp"},
1311         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1312         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1313         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1314         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1315         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1316     },
1317                 "output_path": "/tmp",
1318                 "priority": 1,
1319                 "runtime_constraints": {}
1320         }`
1321
1322         extraMounts := []string{
1323                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1324                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1325                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1326         }
1327
1328         api, runner, realtemp := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1329                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1330                 t.logWriter.Close()
1331         })
1332
1333         c.Check(runner.Binds, DeepEquals, []string{realtemp + "/2:/tmp",
1334                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt:/tmp/foo/bar:ro",
1335                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt:/tmp/foo/baz/sub2file2:ro",
1336                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1:/tmp/foo/sub1:ro",
1337                 realtemp + "/keep1/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt:/tmp/foo/sub1file2:ro",
1338         })
1339
1340         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1341         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1342         for _, v := range api.Content {
1343                 if v["collection"] != nil {
1344                         c.Check(v["ensure_unique_name"], Equals, true)
1345                         collection := v["collection"].(arvadosclient.Dict)
1346                         if strings.Index(collection["name"].(string), "output") == 0 {
1347                                 manifest := collection["manifest_text"].(string)
1348
1349                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1350 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 9:18:bar 9:18:sub1file2
1351 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1352 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1353 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1354 `)
1355                         }
1356                 }
1357         }
1358 }
1359
1360 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1361         helperRecord := `{
1362                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1363                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1364                 "cwd": "/bin",
1365                 "environment": {"FROBIZ": "bilbo"},
1366                 "mounts": {
1367         "/tmp": {"kind": "tmp"},
1368         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt"},
1369         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1370     },
1371                 "output_path": "/tmp",
1372                 "priority": 1,
1373                 "runtime_constraints": {}
1374         }`
1375
1376         extraMounts := []string{
1377                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1378         }
1379
1380         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1381                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1382                 t.logWriter.Close()
1383         })
1384
1385         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1386         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1387         for _, v := range api.Content {
1388                 if v["collection"] != nil {
1389                         collection := v["collection"].(arvadosclient.Dict)
1390                         if strings.Index(collection["name"].(string), "output") == 0 {
1391                                 manifest := collection["manifest_text"].(string)
1392
1393                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1394 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1395 `)
1396                         }
1397                 }
1398         }
1399 }
1400
1401 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1402         helperRecord := `{
1403                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1404                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1405                 "cwd": "/bin",
1406                 "environment": {"FROBIZ": "bilbo"},
1407                 "mounts": {
1408         "/tmp": {"kind": "tmp"},
1409         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1410         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1411     },
1412                 "output_path": "/tmp",
1413                 "priority": 1,
1414                 "runtime_constraints": {}
1415         }`
1416
1417         extraMounts := []string{
1418                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1419         }
1420
1421         api, _, _ := FullRunHelper(c, helperRecord, extraMounts, 0, func(t *TestDockerClient) {
1422                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1423                 t.logWriter.Close()
1424         })
1425
1426         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1427         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1428         for _, v := range api.Content {
1429                 if v["collection"] != nil {
1430                         collection := v["collection"].(arvadosclient.Dict)
1431                         if strings.Index(collection["name"].(string), "output") == 0 {
1432                                 manifest := collection["manifest_text"].(string)
1433
1434                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1435 `)
1436                         }
1437                 }
1438         }
1439 }
1440
1441 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1442         helperRecord := `{
1443                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1444                 "container_image": "d4ab34d3d4f8a72f5c4973051ae69fab+122",
1445                 "cwd": "/bin",
1446                 "environment": {"FROBIZ": "bilbo"},
1447                 "mounts": {
1448         "/tmp": {"kind": "tmp"},
1449         "stdin": {"kind": "json", "content": "foo"},
1450         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1451     },
1452                 "output_path": "/tmp",
1453                 "priority": 1,
1454                 "runtime_constraints": {}
1455         }`
1456
1457         api, _, _ := FullRunHelper(c, helperRecord, nil, 0, func(t *TestDockerClient) {
1458                 t.logWriter.Write(dockerLog(1, t.env[0][7:]+"\n"))
1459                 t.logWriter.Close()
1460         })
1461
1462         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1463         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1464         for _, v := range api.Content {
1465                 if v["collection"] != nil {
1466                         collection := v["collection"].(arvadosclient.Dict)
1467                         if strings.Index(collection["name"].(string), "output") == 0 {
1468                                 manifest := collection["manifest_text"].(string)
1469
1470                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1471 `)
1472                         }
1473                 }
1474         }
1475 }