Merge branch '18665-vscode-setup' refs #18665
[arvados.git] / lib / crunchrun / crunchrun_test.go
1 // Copyright (C) The Arvados Authors. All rights reserved.
2 //
3 // SPDX-License-Identifier: AGPL-3.0
4
5 package crunchrun
6
7 import (
8         "bytes"
9         "crypto/md5"
10         "encoding/json"
11         "errors"
12         "fmt"
13         "io"
14         "io/ioutil"
15         "os"
16         "os/exec"
17         "regexp"
18         "runtime/pprof"
19         "strings"
20         "sync"
21         "syscall"
22         "testing"
23         "time"
24
25         "git.arvados.org/arvados.git/sdk/go/arvados"
26         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
27         "git.arvados.org/arvados.git/sdk/go/arvadostest"
28         "git.arvados.org/arvados.git/sdk/go/manifest"
29         "golang.org/x/net/context"
30
31         . "gopkg.in/check.v1"
32 )
33
34 // Gocheck boilerplate
35 func TestCrunchExec(t *testing.T) {
36         TestingT(t)
37 }
38
39 var _ = Suite(&TestSuite{})
40
41 type TestSuite struct {
42         client                   *arvados.Client
43         api                      *ArvTestClient
44         runner                   *ContainerRunner
45         executor                 *stubExecutor
46         keepmount                string
47         testDispatcherKeepClient KeepTestClient
48         testContainerKeepClient  KeepTestClient
49 }
50
51 func (s *TestSuite) SetUpTest(c *C) {
52         *brokenNodeHook = ""
53         s.client = arvados.NewClientFromEnv()
54         s.executor = &stubExecutor{}
55         var err error
56         s.api = &ArvTestClient{}
57         s.runner, err = NewContainerRunner(s.client, s.api, &s.testDispatcherKeepClient, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
58         c.Assert(err, IsNil)
59         s.runner.executor = s.executor
60         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
61                 return s.api, &s.testContainerKeepClient, s.client, nil
62         }
63         s.runner.RunArvMount = func(cmd []string, tok string) (*exec.Cmd, error) {
64                 s.runner.ArvMountPoint = s.keepmount
65                 return nil, nil
66         }
67         s.keepmount = c.MkDir()
68         err = os.Mkdir(s.keepmount+"/by_id", 0755)
69         c.Assert(err, IsNil)
70         err = os.Mkdir(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH, 0755)
71         c.Assert(err, IsNil)
72         err = ioutil.WriteFile(s.keepmount+"/by_id/"+arvadostest.DockerImage112PDH+"/"+arvadostest.DockerImage112Filename, []byte("#notarealtarball"), 0644)
73         err = os.Mkdir(s.keepmount+"/by_id/"+fakeInputCollectionPDH, 0755)
74         c.Assert(err, IsNil)
75         err = ioutil.WriteFile(s.keepmount+"/by_id/"+fakeInputCollectionPDH+"/input.json", []byte(`{"input":true}`), 0644)
76         c.Assert(err, IsNil)
77         s.runner.ArvMountPoint = s.keepmount
78 }
79
80 type ArvTestClient struct {
81         Total   int64
82         Calls   int
83         Content []arvadosclient.Dict
84         arvados.Container
85         secretMounts []byte
86         Logs         map[string]*bytes.Buffer
87         sync.Mutex
88         WasSetRunning bool
89         callraw       bool
90 }
91
92 type KeepTestClient struct {
93         Called         bool
94         Content        []byte
95         StorageClasses []string
96 }
97
98 type stubExecutor struct {
99         imageLoaded bool
100         loaded      string
101         loadErr     error
102         exitCode    int
103         createErr   error
104         created     containerSpec
105         startErr    error
106         waitSleep   time.Duration
107         waitErr     error
108         stopErr     error
109         stopped     bool
110         closed      bool
111         runFunc     func()
112         exit        chan int
113 }
114
115 func (e *stubExecutor) LoadImage(imageId string, tarball string, container arvados.Container, keepMount string,
116         containerClient *arvados.Client) error {
117         e.loaded = tarball
118         return e.loadErr
119 }
120 func (e *stubExecutor) Runtime() string                 { return "stub" }
121 func (e *stubExecutor) Create(spec containerSpec) error { e.created = spec; return e.createErr }
122 func (e *stubExecutor) Start() error                    { e.exit = make(chan int, 1); go e.runFunc(); return e.startErr }
123 func (e *stubExecutor) CgroupID() string                { return "cgroupid" }
124 func (e *stubExecutor) Stop() error                     { e.stopped = true; go func() { e.exit <- -1 }(); return e.stopErr }
125 func (e *stubExecutor) Close()                          { e.closed = true }
126 func (e *stubExecutor) Wait(context.Context) (int, error) {
127         return <-e.exit, e.waitErr
128 }
129
130 const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
131
132 var hwManifest = ". 82ab40c24fc8df01798e57ba66795bb1+841216+Aa124ac75e5168396c73c0a18eda641a4f41791c0@569fa8c3 0:841216:9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7.tar\n"
133 var hwPDH = "a45557269dcb65a6b78f9ac061c0850b+120"
134 var hwImageID = "9c31ee32b3d15268a0754e8edc74d4f815ee014b693bc5109058e431dd5caea7"
135
136 var otherManifest = ". 68a84f561b1d1708c6baff5e019a9ab3+46+Ae5d0af96944a3690becb1decdf60cc1c937f556d@5693216f 0:46:md5sum.txt\n"
137 var otherPDH = "a3e8f74c6f101eae01fa08bfb4e49b3a+54"
138
139 var normalizedManifestWithSubdirs = `. 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt
140 ./subdir1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
141 ./subdir1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
142 `
143
144 var normalizedWithSubdirsPDH = "a0def87f80dd594d4675809e83bd4f15+367"
145
146 var denormalizedManifestWithSubdirs = ". 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 0:9:file1_in_main.txt 9:18:file2_in_main.txt 0:27:zzzzz-8i9sb-bcdefghijkdhvnk.log.txt 0:10:subdir1/file1_in_subdir1.txt 10:17:subdir1/file2_in_subdir1.txt\n"
147 var denormalizedWithSubdirsPDH = "b0def87f80dd594d4675809e83bd4f15+367"
148
149 var fakeAuthUUID = "zzzzz-gj3su-55pqoyepgi2glem"
150 var fakeAuthToken = "a3ltuwzqcu2u4sc0q7yhpc2w7s00fdcqecg5d6e0u3pfohmbjt"
151
152 func (client *ArvTestClient) Create(resourceType string,
153         parameters arvadosclient.Dict,
154         output interface{}) error {
155
156         client.Mutex.Lock()
157         defer client.Mutex.Unlock()
158
159         client.Calls++
160         client.Content = append(client.Content, parameters)
161
162         if resourceType == "logs" {
163                 et := parameters["log"].(arvadosclient.Dict)["event_type"].(string)
164                 if client.Logs == nil {
165                         client.Logs = make(map[string]*bytes.Buffer)
166                 }
167                 if client.Logs[et] == nil {
168                         client.Logs[et] = &bytes.Buffer{}
169                 }
170                 client.Logs[et].Write([]byte(parameters["log"].(arvadosclient.Dict)["properties"].(map[string]string)["text"]))
171         }
172
173         if resourceType == "collections" && output != nil {
174                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
175                 outmap := output.(*arvados.Collection)
176                 outmap.PortableDataHash = fmt.Sprintf("%x+%d", md5.Sum([]byte(mt)), len(mt))
177                 outmap.UUID = fmt.Sprintf("zzzzz-4zz18-%15.15x", md5.Sum([]byte(mt)))
178         }
179
180         return nil
181 }
182
183 func (client *ArvTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
184         switch {
185         case method == "GET" && resourceType == "containers" && action == "auth":
186                 return json.Unmarshal([]byte(`{
187                         "kind": "arvados#api_client_authorization",
188                         "uuid": "`+fakeAuthUUID+`",
189                         "api_token": "`+fakeAuthToken+`"
190                         }`), output)
191         case method == "GET" && resourceType == "containers" && action == "secret_mounts":
192                 if client.secretMounts != nil {
193                         return json.Unmarshal(client.secretMounts, output)
194                 }
195                 return json.Unmarshal([]byte(`{"secret_mounts":{}}`), output)
196         default:
197                 return fmt.Errorf("Not found")
198         }
199 }
200
201 func (client *ArvTestClient) CallRaw(method, resourceType, uuid, action string,
202         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
203         var j []byte
204         if method == "GET" && resourceType == "nodes" && uuid == "" && action == "" {
205                 j = []byte(`{
206                         "kind": "arvados#nodeList",
207                         "items": [{
208                                 "uuid": "zzzzz-7ekkf-2z3mc76g2q73aio",
209                                 "hostname": "compute2",
210                                 "properties": {"total_cpu_cores": 16}
211                         }]}`)
212         } else if method == "GET" && resourceType == "containers" && action == "" && !client.callraw {
213                 if uuid == "" {
214                         j, err = json.Marshal(map[string]interface{}{
215                                 "items": []interface{}{client.Container},
216                                 "kind":  "arvados#nodeList",
217                         })
218                 } else {
219                         j, err = json.Marshal(client.Container)
220                 }
221         } else {
222                 j = []byte(`{
223                         "command": ["sleep", "1"],
224                         "container_image": "` + arvadostest.DockerImage112PDH + `",
225                         "cwd": ".",
226                         "environment": {},
227                         "mounts": {"/tmp": {"kind": "tmp"}, "/json": {"kind": "json", "content": {"number": 123456789123456789}}},
228                         "output_path": "/tmp",
229                         "priority": 1,
230                         "runtime_constraints": {}
231                 }`)
232         }
233         return ioutil.NopCloser(bytes.NewReader(j)), err
234 }
235
236 func (client *ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
237         if resourceType == "collections" {
238                 if uuid == hwPDH {
239                         output.(*arvados.Collection).ManifestText = hwManifest
240                 } else if uuid == otherPDH {
241                         output.(*arvados.Collection).ManifestText = otherManifest
242                 } else if uuid == normalizedWithSubdirsPDH {
243                         output.(*arvados.Collection).ManifestText = normalizedManifestWithSubdirs
244                 } else if uuid == denormalizedWithSubdirsPDH {
245                         output.(*arvados.Collection).ManifestText = denormalizedManifestWithSubdirs
246                 }
247         }
248         if resourceType == "containers" {
249                 (*output.(*arvados.Container)) = client.Container
250         }
251         return nil
252 }
253
254 func (client *ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
255         client.Mutex.Lock()
256         defer client.Mutex.Unlock()
257         client.Calls++
258         client.Content = append(client.Content, parameters)
259         if resourceType == "containers" {
260                 if parameters["container"].(arvadosclient.Dict)["state"] == "Running" {
261                         client.WasSetRunning = true
262                 }
263         } else if resourceType == "collections" {
264                 mt := parameters["collection"].(arvadosclient.Dict)["manifest_text"].(string)
265                 output.(*arvados.Collection).UUID = uuid
266                 output.(*arvados.Collection).PortableDataHash = fmt.Sprintf("%x", md5.Sum([]byte(mt)))
267         }
268         return nil
269 }
270
271 var discoveryMap = map[string]interface{}{
272         "defaultTrashLifetime":               float64(1209600),
273         "crunchLimitLogBytesPerJob":          float64(67108864),
274         "crunchLogThrottleBytes":             float64(65536),
275         "crunchLogThrottlePeriod":            float64(60),
276         "crunchLogThrottleLines":             float64(1024),
277         "crunchLogPartialLineThrottlePeriod": float64(5),
278         "crunchLogBytesPerEvent":             float64(4096),
279         "crunchLogSecondsBetweenEvents":      float64(1),
280 }
281
282 func (client *ArvTestClient) Discovery(key string) (interface{}, error) {
283         return discoveryMap[key], nil
284 }
285
286 // CalledWith returns the parameters from the first API call whose
287 // parameters match jpath/string. E.g., CalledWith(c, "foo.bar",
288 // "baz") returns parameters with parameters["foo"]["bar"]=="baz". If
289 // no call matches, it returns nil.
290 func (client *ArvTestClient) CalledWith(jpath string, expect interface{}) arvadosclient.Dict {
291 call:
292         for _, content := range client.Content {
293                 var v interface{} = content
294                 for _, k := range strings.Split(jpath, ".") {
295                         if dict, ok := v.(arvadosclient.Dict); !ok {
296                                 continue call
297                         } else {
298                                 v = dict[k]
299                         }
300                 }
301                 if v == expect {
302                         return content
303                 }
304         }
305         return nil
306 }
307
308 func (client *KeepTestClient) LocalLocator(locator string) (string, error) {
309         return locator, nil
310 }
311
312 func (client *KeepTestClient) BlockWrite(_ context.Context, opts arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
313         client.Content = opts.Data
314         return arvados.BlockWriteResponse{
315                 Locator: fmt.Sprintf("%x+%d", md5.Sum(opts.Data), len(opts.Data)),
316         }, nil
317 }
318
319 func (client *KeepTestClient) ReadAt(string, []byte, int) (int, error) {
320         return 0, errors.New("not implemented")
321 }
322
323 func (client *KeepTestClient) ClearBlockCache() {
324 }
325
326 func (client *KeepTestClient) Close() {
327         client.Content = nil
328 }
329
330 func (client *KeepTestClient) SetStorageClasses(sc []string) {
331         client.StorageClasses = sc
332 }
333
334 type FileWrapper struct {
335         io.ReadCloser
336         len int64
337 }
338
339 func (fw FileWrapper) Readdir(n int) ([]os.FileInfo, error) {
340         return nil, errors.New("not implemented")
341 }
342
343 func (fw FileWrapper) Seek(int64, int) (int64, error) {
344         return 0, errors.New("not implemented")
345 }
346
347 func (fw FileWrapper) Size() int64 {
348         return fw.len
349 }
350
351 func (fw FileWrapper) Stat() (os.FileInfo, error) {
352         return nil, errors.New("not implemented")
353 }
354
355 func (fw FileWrapper) Truncate(int64) error {
356         return errors.New("not implemented")
357 }
358
359 func (fw FileWrapper) Write([]byte) (int, error) {
360         return 0, errors.New("not implemented")
361 }
362
363 func (fw FileWrapper) Sync() error {
364         return errors.New("not implemented")
365 }
366
367 func (client *KeepTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
368         if filename == hwImageID+".tar" {
369                 rdr := ioutil.NopCloser(&bytes.Buffer{})
370                 client.Called = true
371                 return FileWrapper{rdr, 1321984}, nil
372         } else if filename == "/file1_in_main.txt" {
373                 rdr := ioutil.NopCloser(strings.NewReader("foo"))
374                 client.Called = true
375                 return FileWrapper{rdr, 3}, nil
376         }
377         return nil, nil
378 }
379
380 func (s *TestSuite) TestLoadImage(c *C) {
381         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
382         s.runner.Container.Mounts = map[string]arvados.Mount{
383                 "/out": {Kind: "tmp", Writable: true},
384         }
385         s.runner.Container.OutputPath = "/out"
386
387         _, err := s.runner.SetupMounts()
388         c.Assert(err, IsNil)
389
390         imageID, err := s.runner.LoadImage()
391         c.Check(err, IsNil)
392         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
393         c.Check(imageID, Equals, strings.TrimSuffix(arvadostest.DockerImage112Filename, ".tar"))
394
395         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
396         s.executor.imageLoaded = false
397         s.executor.loaded = ""
398         s.executor.loadErr = errors.New("bork")
399         imageID, err = s.runner.LoadImage()
400         c.Check(err, ErrorMatches, ".*bork")
401         c.Check(s.executor.loaded, Matches, ".*"+regexp.QuoteMeta(arvadostest.DockerImage112Filename))
402
403         s.runner.Container.ContainerImage = fakeInputCollectionPDH
404         s.executor.imageLoaded = false
405         s.executor.loaded = ""
406         s.executor.loadErr = nil
407         imageID, err = s.runner.LoadImage()
408         c.Check(err, ErrorMatches, "image collection does not include a \\.tar image file")
409         c.Check(s.executor.loaded, Equals, "")
410 }
411
412 type ArvErrorTestClient struct{}
413
414 func (ArvErrorTestClient) Create(resourceType string,
415         parameters arvadosclient.Dict,
416         output interface{}) error {
417         return nil
418 }
419
420 func (ArvErrorTestClient) Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error {
421         if method == "GET" && resourceType == "containers" && action == "auth" {
422                 return nil
423         }
424         return errors.New("ArvError")
425 }
426
427 func (ArvErrorTestClient) CallRaw(method, resourceType, uuid, action string,
428         parameters arvadosclient.Dict) (reader io.ReadCloser, err error) {
429         return nil, errors.New("ArvError")
430 }
431
432 func (ArvErrorTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error {
433         return errors.New("ArvError")
434 }
435
436 func (ArvErrorTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
437         return nil
438 }
439
440 func (ArvErrorTestClient) Discovery(key string) (interface{}, error) {
441         return discoveryMap[key], nil
442 }
443
444 type KeepErrorTestClient struct {
445         KeepTestClient
446 }
447
448 func (*KeepErrorTestClient) ManifestFileReader(manifest.Manifest, string) (arvados.File, error) {
449         return nil, errors.New("KeepError")
450 }
451
452 func (*KeepErrorTestClient) BlockWrite(context.Context, arvados.BlockWriteOptions) (arvados.BlockWriteResponse, error) {
453         return arvados.BlockWriteResponse{}, errors.New("KeepError")
454 }
455
456 func (*KeepErrorTestClient) LocalLocator(string) (string, error) {
457         return "", errors.New("KeepError")
458 }
459
460 type KeepReadErrorTestClient struct {
461         KeepTestClient
462 }
463
464 func (*KeepReadErrorTestClient) ReadAt(string, []byte, int) (int, error) {
465         return 0, errors.New("KeepError")
466 }
467
468 type ErrorReader struct {
469         FileWrapper
470 }
471
472 func (ErrorReader) Read(p []byte) (n int, err error) {
473         return 0, errors.New("ErrorReader")
474 }
475
476 func (ErrorReader) Seek(int64, int) (int64, error) {
477         return 0, errors.New("ErrorReader")
478 }
479
480 func (KeepReadErrorTestClient) ManifestFileReader(m manifest.Manifest, filename string) (arvados.File, error) {
481         return ErrorReader{}, nil
482 }
483
484 type ClosableBuffer struct {
485         bytes.Buffer
486 }
487
488 func (*ClosableBuffer) Close() error {
489         return nil
490 }
491
492 type TestLogs struct {
493         Stdout ClosableBuffer
494         Stderr ClosableBuffer
495 }
496
497 func (tl *TestLogs) NewTestLoggingWriter(logstr string) (io.WriteCloser, error) {
498         if logstr == "stdout" {
499                 return &tl.Stdout, nil
500         }
501         if logstr == "stderr" {
502                 return &tl.Stderr, nil
503         }
504         return nil, errors.New("???")
505 }
506
507 func dockerLog(fd byte, msg string) []byte {
508         by := []byte(msg)
509         header := make([]byte, 8+len(by))
510         header[0] = fd
511         header[7] = byte(len(by))
512         copy(header[8:], by)
513         return header
514 }
515
516 func (s *TestSuite) TestRunContainer(c *C) {
517         s.executor.runFunc = func() {
518                 fmt.Fprintf(s.executor.created.Stdout, "Hello world\n")
519                 s.executor.exit <- 0
520         }
521
522         var logs TestLogs
523         s.runner.NewLogWriter = logs.NewTestLoggingWriter
524         s.runner.Container.ContainerImage = arvadostest.DockerImage112PDH
525         s.runner.Container.Command = []string{"./hw"}
526         s.runner.Container.OutputStorageClasses = []string{"default"}
527
528         imageID, err := s.runner.LoadImage()
529         c.Assert(err, IsNil)
530
531         err = s.runner.CreateContainer(imageID, nil)
532         c.Assert(err, IsNil)
533
534         err = s.runner.StartContainer()
535         c.Assert(err, IsNil)
536
537         err = s.runner.WaitFinish()
538         c.Assert(err, IsNil)
539
540         c.Check(logs.Stdout.String(), Matches, ".*Hello world\n")
541         c.Check(logs.Stderr.String(), Equals, "")
542 }
543
544 func (s *TestSuite) TestCommitLogs(c *C) {
545         api := &ArvTestClient{}
546         kc := &KeepTestClient{}
547         defer kc.Close()
548         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
549         c.Assert(err, IsNil)
550         cr.CrunchLog.Timestamper = (&TestTimestamper{}).Timestamp
551
552         cr.CrunchLog.Print("Hello world!")
553         cr.CrunchLog.Print("Goodbye")
554         cr.finalState = "Complete"
555
556         err = cr.CommitLogs()
557         c.Check(err, IsNil)
558
559         c.Check(api.Calls, Equals, 2)
560         c.Check(api.Content[1]["ensure_unique_name"], Equals, true)
561         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["name"], Equals, "logs for zzzzz-zzzzz-zzzzzzzzzzzzzzz")
562         c.Check(api.Content[1]["collection"].(arvadosclient.Dict)["manifest_text"], Equals, ". 744b2e4553123b02fa7b452ec5c18993+123 0:123:crunch-run.txt\n")
563         c.Check(*cr.LogsPDH, Equals, "63da7bdacf08c40f604daad80c261e9a+60")
564 }
565
566 func (s *TestSuite) TestUpdateContainerRunning(c *C) {
567         api := &ArvTestClient{}
568         kc := &KeepTestClient{}
569         defer kc.Close()
570         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
571         c.Assert(err, IsNil)
572
573         err = cr.UpdateContainerRunning()
574         c.Check(err, IsNil)
575
576         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Running")
577 }
578
579 func (s *TestSuite) TestUpdateContainerComplete(c *C) {
580         api := &ArvTestClient{}
581         kc := &KeepTestClient{}
582         defer kc.Close()
583         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
584         c.Assert(err, IsNil)
585
586         cr.LogsPDH = new(string)
587         *cr.LogsPDH = "d3a229d2fe3690c2c3e75a71a153c6a3+60"
588
589         cr.ExitCode = new(int)
590         *cr.ExitCode = 42
591         cr.finalState = "Complete"
592
593         err = cr.UpdateContainerFinal()
594         c.Check(err, IsNil)
595
596         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], Equals, *cr.LogsPDH)
597         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], Equals, *cr.ExitCode)
598         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Complete")
599 }
600
601 func (s *TestSuite) TestUpdateContainerCancelled(c *C) {
602         api := &ArvTestClient{}
603         kc := &KeepTestClient{}
604         defer kc.Close()
605         cr, err := NewContainerRunner(s.client, api, kc, "zzzzz-zzzzz-zzzzzzzzzzzzzzz")
606         c.Assert(err, IsNil)
607         cr.cCancelled = true
608         cr.finalState = "Cancelled"
609
610         err = cr.UpdateContainerFinal()
611         c.Check(err, IsNil)
612
613         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["log"], IsNil)
614         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["exit_code"], IsNil)
615         c.Check(api.Content[0]["container"].(arvadosclient.Dict)["state"], Equals, "Cancelled")
616 }
617
618 // Used by the TestFullRun*() test below to DRY up boilerplate setup to do full
619 // dress rehearsal of the Run() function, starting from a JSON container record.
620 func (s *TestSuite) fullRunHelper(c *C, record string, extraMounts []string, exitCode int, fn func()) (*ArvTestClient, *ContainerRunner, string) {
621         err := json.Unmarshal([]byte(record), &s.api.Container)
622         c.Assert(err, IsNil)
623         initialState := s.api.Container.State
624
625         var sm struct {
626                 SecretMounts map[string]arvados.Mount `json:"secret_mounts"`
627         }
628         err = json.Unmarshal([]byte(record), &sm)
629         c.Check(err, IsNil)
630         secretMounts, err := json.Marshal(sm)
631         c.Assert(err, IsNil)
632         c.Logf("SecretMounts decoded %v json %q", sm, secretMounts)
633
634         s.executor.runFunc = func() {
635                 fn()
636                 s.executor.exit <- exitCode
637         }
638
639         s.runner.statInterval = 100 * time.Millisecond
640         s.runner.containerWatchdogInterval = time.Second
641         am := &ArvMountCmdLine{}
642         s.runner.RunArvMount = am.ArvMountTest
643
644         realTemp := c.MkDir()
645         tempcount := 0
646         s.runner.MkTempDir = func(_, prefix string) (string, error) {
647                 tempcount++
648                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, tempcount)
649                 err := os.Mkdir(d, os.ModePerm)
650                 if err != nil && strings.Contains(err.Error(), ": file exists") {
651                         // Test case must have pre-populated the tempdir
652                         err = nil
653                 }
654                 return d, err
655         }
656         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
657                 return &ArvTestClient{secretMounts: secretMounts}, &s.testContainerKeepClient, nil, nil
658         }
659
660         if extraMounts != nil && len(extraMounts) > 0 {
661                 err := s.runner.SetupArvMountPoint("keep")
662                 c.Check(err, IsNil)
663
664                 for _, m := range extraMounts {
665                         os.MkdirAll(s.runner.ArvMountPoint+"/by_id/"+m, os.ModePerm)
666                 }
667         }
668
669         err = s.runner.Run()
670         if s.api.CalledWith("container.state", "Complete") != nil {
671                 c.Check(err, IsNil)
672         }
673         if s.executor.loadErr == nil && s.executor.createErr == nil && initialState != "Running" {
674                 c.Check(s.api.WasSetRunning, Equals, true)
675                 var lastupdate arvadosclient.Dict
676                 for _, content := range s.api.Content {
677                         if content["container"] != nil {
678                                 lastupdate = content["container"].(arvadosclient.Dict)
679                         }
680                 }
681                 if lastupdate["log"] == nil {
682                         c.Errorf("no container update with non-nil log -- updates were: %v", s.api.Content)
683                 }
684         }
685
686         if err != nil {
687                 for k, v := range s.api.Logs {
688                         c.Log(k)
689                         c.Log(v.String())
690                 }
691         }
692
693         return s.api, s.runner, realTemp
694 }
695
696 func (s *TestSuite) TestFullRunHello(c *C) {
697         s.runner.enableMemoryLimit = true
698         s.runner.networkMode = "default"
699         s.fullRunHelper(c, `{
700     "command": ["echo", "hello world"],
701     "container_image": "`+arvadostest.DockerImage112PDH+`",
702     "cwd": ".",
703     "environment": {"foo":"bar","baz":"waz"},
704     "mounts": {"/tmp": {"kind": "tmp"} },
705     "output_path": "/tmp",
706     "priority": 1,
707     "runtime_constraints": {"vcpus":1,"ram":1000000},
708     "state": "Locked",
709     "output_storage_classes": ["default"]
710 }`, nil, 0, func() {
711                 c.Check(s.executor.created.Command, DeepEquals, []string{"echo", "hello world"})
712                 c.Check(s.executor.created.Image, Equals, "sha256:d8309758b8fe2c81034ffc8a10c36460b77db7bc5e7b448c4e5b684f9d95a678")
713                 c.Check(s.executor.created.Env, DeepEquals, map[string]string{"foo": "bar", "baz": "waz"})
714                 c.Check(s.executor.created.VCPUs, Equals, 1)
715                 c.Check(s.executor.created.RAM, Equals, int64(1000000))
716                 c.Check(s.executor.created.NetworkMode, Equals, "default")
717                 c.Check(s.executor.created.EnableNetwork, Equals, false)
718                 c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
719                 fmt.Fprintln(s.executor.created.Stdout, "hello world")
720         })
721
722         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
723         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
724         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello world\n")
725         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"default"})
726         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"default"})
727 }
728
729 func (s *TestSuite) TestRunAlreadyRunning(c *C) {
730         var ran bool
731         s.fullRunHelper(c, `{
732     "command": ["sleep", "3"],
733     "container_image": "`+arvadostest.DockerImage112PDH+`",
734     "cwd": ".",
735     "environment": {},
736     "mounts": {"/tmp": {"kind": "tmp"} },
737     "output_path": "/tmp",
738     "priority": 1,
739     "runtime_constraints": {},
740     "scheduling_parameters":{"max_run_time": 1},
741     "state": "Running"
742 }`, nil, 2, func() {
743                 ran = true
744         })
745         c.Check(s.api.CalledWith("container.state", "Cancelled"), IsNil)
746         c.Check(s.api.CalledWith("container.state", "Complete"), IsNil)
747         c.Check(ran, Equals, false)
748 }
749
750 func (s *TestSuite) TestRunTimeExceeded(c *C) {
751         s.fullRunHelper(c, `{
752     "command": ["sleep", "3"],
753     "container_image": "`+arvadostest.DockerImage112PDH+`",
754     "cwd": ".",
755     "environment": {},
756     "mounts": {"/tmp": {"kind": "tmp"} },
757     "output_path": "/tmp",
758     "priority": 1,
759     "runtime_constraints": {},
760     "scheduling_parameters":{"max_run_time": 1},
761     "state": "Locked"
762 }`, nil, 0, func() {
763                 time.Sleep(3 * time.Second)
764         })
765
766         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
767         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*maximum run time exceeded.*")
768 }
769
770 func (s *TestSuite) TestContainerWaitFails(c *C) {
771         s.fullRunHelper(c, `{
772     "command": ["sleep", "3"],
773     "container_image": "`+arvadostest.DockerImage112PDH+`",
774     "cwd": ".",
775     "mounts": {"/tmp": {"kind": "tmp"} },
776     "output_path": "/tmp",
777     "priority": 1,
778     "state": "Locked"
779 }`, nil, 0, func() {
780                 s.executor.waitErr = errors.New("Container is not running")
781         })
782
783         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
784         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Container is not running.*")
785 }
786
787 func (s *TestSuite) TestCrunchstat(c *C) {
788         s.fullRunHelper(c, `{
789                 "command": ["sleep", "1"],
790                 "container_image": "`+arvadostest.DockerImage112PDH+`",
791                 "cwd": ".",
792                 "environment": {},
793                 "mounts": {"/tmp": {"kind": "tmp"} },
794                 "output_path": "/tmp",
795                 "priority": 1,
796                 "runtime_constraints": {},
797                 "state": "Locked"
798         }`, nil, 0, func() {
799                 time.Sleep(time.Second)
800         })
801
802         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
803         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
804
805         // We didn't actually start a container, so crunchstat didn't
806         // find accounting files and therefore didn't log any stats.
807         // It should have logged a "can't find accounting files"
808         // message after one poll interval, though, so we can confirm
809         // it's alive:
810         c.Assert(s.api.Logs["crunchstat"], NotNil)
811         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files have not appeared after 100ms.*`)
812
813         // The "files never appeared" log assures us that we called
814         // (*crunchstat.Reporter)Stop(), and that we set it up with
815         // the correct container ID "abcde":
816         c.Check(s.api.Logs["crunchstat"].String(), Matches, `(?ms).*cgroup stats files never appeared for cgroupid\n`)
817 }
818
819 func (s *TestSuite) TestNodeInfoLog(c *C) {
820         os.Setenv("SLURMD_NODENAME", "compute2")
821         s.fullRunHelper(c, `{
822                 "command": ["sleep", "1"],
823                 "container_image": "`+arvadostest.DockerImage112PDH+`",
824                 "cwd": ".",
825                 "environment": {},
826                 "mounts": {"/tmp": {"kind": "tmp"} },
827                 "output_path": "/tmp",
828                 "priority": 1,
829                 "runtime_constraints": {},
830                 "state": "Locked"
831         }`, nil, 0,
832                 func() {
833                         time.Sleep(time.Second)
834                 })
835
836         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
837         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
838
839         c.Assert(s.api.Logs["node"], NotNil)
840         json := s.api.Logs["node"].String()
841         c.Check(json, Matches, `(?ms).*"uuid": *"zzzzz-7ekkf-2z3mc76g2q73aio".*`)
842         c.Check(json, Matches, `(?ms).*"total_cpu_cores": *16.*`)
843         c.Check(json, Not(Matches), `(?ms).*"info":.*`)
844
845         c.Assert(s.api.Logs["node-info"], NotNil)
846         json = s.api.Logs["node-info"].String()
847         c.Check(json, Matches, `(?ms).*Host Information.*`)
848         c.Check(json, Matches, `(?ms).*CPU Information.*`)
849         c.Check(json, Matches, `(?ms).*Memory Information.*`)
850         c.Check(json, Matches, `(?ms).*Disk Space.*`)
851         c.Check(json, Matches, `(?ms).*Disk INodes.*`)
852 }
853
854 func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
855         s.fullRunHelper(c, `{
856                 "command": ["sleep", "1"],
857                 "container_image": "`+arvadostest.DockerImage112PDH+`",
858                 "cwd": ".",
859                 "environment": {},
860                 "mounts": {"/tmp": {"kind": "tmp"} },
861                 "output_path": "/tmp",
862                 "priority": 1,
863                 "runtime_constraints": {},
864                 "state": "Locked"
865         }`, nil, 0,
866                 func() {
867                 })
868
869         c.Assert(s.api.Logs["crunch-run"], NotNil)
870         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
871         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
872 }
873
874 func (s *TestSuite) TestContainerRecordLog(c *C) {
875         s.fullRunHelper(c, `{
876                 "command": ["sleep", "1"],
877                 "container_image": "`+arvadostest.DockerImage112PDH+`",
878                 "cwd": ".",
879                 "environment": {},
880                 "mounts": {"/tmp": {"kind": "tmp"} },
881                 "output_path": "/tmp",
882                 "priority": 1,
883                 "runtime_constraints": {},
884                 "state": "Locked"
885         }`, nil, 0,
886                 func() {
887                         time.Sleep(time.Second)
888                 })
889
890         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
891         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
892
893         c.Assert(s.api.Logs["container"], NotNil)
894         c.Check(s.api.Logs["container"].String(), Matches, `(?ms).*container_image.*`)
895 }
896
897 func (s *TestSuite) TestFullRunStderr(c *C) {
898         s.fullRunHelper(c, `{
899     "command": ["/bin/sh", "-c", "echo hello ; echo world 1>&2 ; exit 1"],
900     "container_image": "`+arvadostest.DockerImage112PDH+`",
901     "cwd": ".",
902     "environment": {},
903     "mounts": {"/tmp": {"kind": "tmp"} },
904     "output_path": "/tmp",
905     "priority": 1,
906     "runtime_constraints": {},
907     "state": "Locked"
908 }`, nil, 1, func() {
909                 fmt.Fprintln(s.executor.created.Stdout, "hello")
910                 fmt.Fprintln(s.executor.created.Stderr, "world")
911         })
912
913         final := s.api.CalledWith("container.state", "Complete")
914         c.Assert(final, NotNil)
915         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
916         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
917
918         c.Check(s.api.Logs["stdout"].String(), Matches, ".*hello\n")
919         c.Check(s.api.Logs["stderr"].String(), Matches, ".*world\n")
920 }
921
922 func (s *TestSuite) TestFullRunDefaultCwd(c *C) {
923         s.fullRunHelper(c, `{
924     "command": ["pwd"],
925     "container_image": "`+arvadostest.DockerImage112PDH+`",
926     "cwd": ".",
927     "environment": {},
928     "mounts": {"/tmp": {"kind": "tmp"} },
929     "output_path": "/tmp",
930     "priority": 1,
931     "runtime_constraints": {},
932     "state": "Locked"
933 }`, nil, 0, func() {
934                 fmt.Fprintf(s.executor.created.Stdout, "workdir=%q", s.executor.created.WorkingDir)
935         })
936
937         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
938         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
939         c.Log(s.api.Logs["stdout"])
940         c.Check(s.api.Logs["stdout"].String(), Matches, `.*workdir=""\n`)
941 }
942
943 func (s *TestSuite) TestFullRunSetCwd(c *C) {
944         s.fullRunHelper(c, `{
945     "command": ["pwd"],
946     "container_image": "`+arvadostest.DockerImage112PDH+`",
947     "cwd": "/bin",
948     "environment": {},
949     "mounts": {"/tmp": {"kind": "tmp"} },
950     "output_path": "/tmp",
951     "priority": 1,
952     "runtime_constraints": {},
953     "state": "Locked"
954 }`, nil, 0, func() {
955                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
956         })
957
958         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
959         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
960         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
961 }
962
963 func (s *TestSuite) TestFullRunSetOutputStorageClasses(c *C) {
964         s.fullRunHelper(c, `{
965     "command": ["pwd"],
966     "container_image": "`+arvadostest.DockerImage112PDH+`",
967     "cwd": "/bin",
968     "environment": {},
969     "mounts": {"/tmp": {"kind": "tmp"} },
970     "output_path": "/tmp",
971     "priority": 1,
972     "runtime_constraints": {},
973     "state": "Locked",
974     "output_storage_classes": ["foo", "bar"]
975 }`, nil, 0, func() {
976                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.WorkingDir)
977         })
978
979         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
980         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
981         c.Check(s.api.Logs["stdout"].String(), Matches, ".*/bin\n")
982         c.Check(s.testDispatcherKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
983         c.Check(s.testContainerKeepClient.StorageClasses, DeepEquals, []string{"foo", "bar"})
984 }
985
986 func (s *TestSuite) TestEnableCUDADeviceCount(c *C) {
987         s.fullRunHelper(c, `{
988     "command": ["pwd"],
989     "container_image": "`+arvadostest.DockerImage112PDH+`",
990     "cwd": "/bin",
991     "environment": {},
992     "mounts": {"/tmp": {"kind": "tmp"} },
993     "output_path": "/tmp",
994     "priority": 1,
995     "runtime_constraints": {"cuda": {"device_count": 2}},
996     "state": "Locked",
997     "output_storage_classes": ["foo", "bar"]
998 }`, nil, 0, func() {
999                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1000         })
1001         c.Check(s.executor.created.CUDADeviceCount, Equals, 2)
1002 }
1003
1004 func (s *TestSuite) TestEnableCUDAHardwareCapability(c *C) {
1005         s.fullRunHelper(c, `{
1006     "command": ["pwd"],
1007     "container_image": "`+arvadostest.DockerImage112PDH+`",
1008     "cwd": "/bin",
1009     "environment": {},
1010     "mounts": {"/tmp": {"kind": "tmp"} },
1011     "output_path": "/tmp",
1012     "priority": 1,
1013     "runtime_constraints": {"cuda": {"hardware_capability": "foo"}},
1014     "state": "Locked",
1015     "output_storage_classes": ["foo", "bar"]
1016 }`, nil, 0, func() {
1017                 fmt.Fprintln(s.executor.created.Stdout, "ok")
1018         })
1019         c.Check(s.executor.created.CUDADeviceCount, Equals, 0)
1020 }
1021
1022 func (s *TestSuite) TestStopOnSignal(c *C) {
1023         s.executor.runFunc = func() {
1024                 s.executor.created.Stdout.Write([]byte("foo\n"))
1025                 s.runner.SigChan <- syscall.SIGINT
1026         }
1027         s.testStopContainer(c)
1028 }
1029
1030 func (s *TestSuite) TestStopOnArvMountDeath(c *C) {
1031         s.executor.runFunc = func() {
1032                 s.executor.created.Stdout.Write([]byte("foo\n"))
1033                 s.runner.ArvMountExit <- nil
1034                 close(s.runner.ArvMountExit)
1035         }
1036         s.runner.ArvMountExit = make(chan error)
1037         s.testStopContainer(c)
1038 }
1039
1040 func (s *TestSuite) testStopContainer(c *C) {
1041         record := `{
1042     "command": ["/bin/sh", "-c", "echo foo && sleep 30 && echo bar"],
1043     "container_image": "` + arvadostest.DockerImage112PDH + `",
1044     "cwd": ".",
1045     "environment": {},
1046     "mounts": {"/tmp": {"kind": "tmp"} },
1047     "output_path": "/tmp",
1048     "priority": 1,
1049     "runtime_constraints": {},
1050     "state": "Locked"
1051 }`
1052
1053         err := json.Unmarshal([]byte(record), &s.api.Container)
1054         c.Assert(err, IsNil)
1055
1056         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) { return nil, nil }
1057         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1058                 return &ArvTestClient{}, &KeepTestClient{}, nil, nil
1059         }
1060
1061         done := make(chan error)
1062         go func() {
1063                 done <- s.runner.Run()
1064         }()
1065         select {
1066         case <-time.After(20 * time.Second):
1067                 pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
1068                 c.Fatal("timed out")
1069         case err = <-done:
1070                 c.Check(err, IsNil)
1071         }
1072         for k, v := range s.api.Logs {
1073                 c.Log(k)
1074                 c.Log(v.String(), "\n")
1075         }
1076
1077         c.Check(s.api.CalledWith("container.log", nil), NotNil)
1078         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1079         c.Check(s.api.Logs["stdout"].String(), Matches, "(?ms).*foo\n$")
1080 }
1081
1082 func (s *TestSuite) TestFullRunSetEnv(c *C) {
1083         s.fullRunHelper(c, `{
1084     "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1085     "container_image": "`+arvadostest.DockerImage112PDH+`",
1086     "cwd": "/bin",
1087     "environment": {"FROBIZ": "bilbo"},
1088     "mounts": {"/tmp": {"kind": "tmp"} },
1089     "output_path": "/tmp",
1090     "priority": 1,
1091     "runtime_constraints": {},
1092     "state": "Locked"
1093 }`, nil, 0, func() {
1094                 fmt.Fprintf(s.executor.created.Stdout, "%v", s.executor.created.Env)
1095         })
1096
1097         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1098         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1099         c.Check(s.api.Logs["stdout"].String(), Matches, `.*map\[FROBIZ:bilbo\]\n`)
1100 }
1101
1102 type ArvMountCmdLine struct {
1103         Cmd   []string
1104         token string
1105 }
1106
1107 func (am *ArvMountCmdLine) ArvMountTest(c []string, token string) (*exec.Cmd, error) {
1108         am.Cmd = c
1109         am.token = token
1110         return nil, nil
1111 }
1112
1113 func stubCert(temp string) string {
1114         path := temp + "/ca-certificates.crt"
1115         crt, _ := os.Create(path)
1116         crt.Close()
1117         arvadosclient.CertFiles = []string{path}
1118         return path
1119 }
1120
1121 func (s *TestSuite) TestSetupMounts(c *C) {
1122         cr := s.runner
1123         am := &ArvMountCmdLine{}
1124         cr.RunArvMount = am.ArvMountTest
1125         cr.ContainerArvClient = &ArvTestClient{}
1126         cr.ContainerKeepClient = &KeepTestClient{}
1127         cr.Container.OutputStorageClasses = []string{"default"}
1128
1129         realTemp := c.MkDir()
1130         certTemp := c.MkDir()
1131         stubCertPath := stubCert(certTemp)
1132         cr.parentTemp = realTemp
1133
1134         i := 0
1135         cr.MkTempDir = func(_ string, prefix string) (string, error) {
1136                 i++
1137                 d := fmt.Sprintf("%s/%s%d", realTemp, prefix, i)
1138                 err := os.Mkdir(d, os.ModePerm)
1139                 if err != nil && strings.Contains(err.Error(), ": file exists") {
1140                         // Test case must have pre-populated the tempdir
1141                         err = nil
1142                 }
1143                 return d, err
1144         }
1145
1146         checkEmpty := func() {
1147                 // Should be deleted.
1148                 _, err := os.Stat(realTemp)
1149                 c.Assert(os.IsNotExist(err), Equals, true)
1150
1151                 // Now recreate it for the next test.
1152                 c.Assert(os.Mkdir(realTemp, 0777), IsNil)
1153         }
1154
1155         {
1156                 i = 0
1157                 cr.ArvMountPoint = ""
1158                 cr.Container.Mounts = make(map[string]arvados.Mount)
1159                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1160                 cr.Container.OutputPath = "/tmp"
1161                 cr.statInterval = 5 * time.Second
1162                 bindmounts, err := cr.SetupMounts()
1163                 c.Check(err, IsNil)
1164                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1165                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1166                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1167                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}})
1168                 os.RemoveAll(cr.ArvMountPoint)
1169                 cr.CleanupDirs()
1170                 checkEmpty()
1171         }
1172
1173         {
1174                 i = 0
1175                 cr.ArvMountPoint = ""
1176                 cr.Container.Mounts = make(map[string]arvados.Mount)
1177                 cr.Container.Mounts["/out"] = arvados.Mount{Kind: "tmp"}
1178                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1179                 cr.Container.OutputPath = "/out"
1180                 cr.Container.OutputStorageClasses = []string{"foo", "bar"}
1181
1182                 bindmounts, err := cr.SetupMounts()
1183                 c.Check(err, IsNil)
1184                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1185                         "--read-write", "--storage-classes", "foo,bar", "--crunchstat-interval=5",
1186                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1187                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/out": {realTemp + "/tmp2", false}, "/tmp": {realTemp + "/tmp3", false}})
1188                 os.RemoveAll(cr.ArvMountPoint)
1189                 cr.CleanupDirs()
1190                 checkEmpty()
1191         }
1192
1193         {
1194                 i = 0
1195                 cr.ArvMountPoint = ""
1196                 cr.Container.Mounts = make(map[string]arvados.Mount)
1197                 cr.Container.Mounts["/tmp"] = arvados.Mount{Kind: "tmp"}
1198                 cr.Container.OutputPath = "/tmp"
1199                 cr.Container.RuntimeConstraints.API = true
1200                 cr.Container.OutputStorageClasses = []string{"default"}
1201
1202                 bindmounts, err := cr.SetupMounts()
1203                 c.Check(err, IsNil)
1204                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1205                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1206                         "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1207                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/tmp": {realTemp + "/tmp2", false}, "/etc/arvados/ca-certificates.crt": {stubCertPath, true}})
1208                 os.RemoveAll(cr.ArvMountPoint)
1209                 cr.CleanupDirs()
1210                 checkEmpty()
1211
1212                 cr.Container.RuntimeConstraints.API = false
1213         }
1214
1215         {
1216                 i = 0
1217                 cr.ArvMountPoint = ""
1218                 cr.Container.Mounts = map[string]arvados.Mount{
1219                         "/keeptmp": {Kind: "collection", Writable: true},
1220                 }
1221                 cr.Container.OutputPath = "/keeptmp"
1222
1223                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1224
1225                 bindmounts, err := cr.SetupMounts()
1226                 c.Check(err, IsNil)
1227                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1228                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1229                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1230                 c.Check(bindmounts, DeepEquals, map[string]bindmount{"/keeptmp": {realTemp + "/keep1/tmp0", false}})
1231                 os.RemoveAll(cr.ArvMountPoint)
1232                 cr.CleanupDirs()
1233                 checkEmpty()
1234         }
1235
1236         {
1237                 i = 0
1238                 cr.ArvMountPoint = ""
1239                 cr.Container.Mounts = map[string]arvados.Mount{
1240                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1241                         "/keepout": {Kind: "collection", Writable: true},
1242                 }
1243                 cr.Container.OutputPath = "/keepout"
1244
1245                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1246                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1247
1248                 bindmounts, err := cr.SetupMounts()
1249                 c.Check(err, IsNil)
1250                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1251                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1252                         "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1253                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1254                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1255                         "/keepout": {realTemp + "/keep1/tmp0", false},
1256                 })
1257                 os.RemoveAll(cr.ArvMountPoint)
1258                 cr.CleanupDirs()
1259                 checkEmpty()
1260         }
1261
1262         {
1263                 i = 0
1264                 cr.ArvMountPoint = ""
1265                 cr.Container.RuntimeConstraints.KeepCacheRAM = 512
1266                 cr.Container.Mounts = map[string]arvados.Mount{
1267                         "/keepinp": {Kind: "collection", PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53"},
1268                         "/keepout": {Kind: "collection", Writable: true},
1269                 }
1270                 cr.Container.OutputPath = "/keepout"
1271
1272                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1273                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1274
1275                 bindmounts, err := cr.SetupMounts()
1276                 c.Check(err, IsNil)
1277                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1278                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1279                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1280                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1281                         "/keepinp": {realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", true},
1282                         "/keepout": {realTemp + "/keep1/tmp0", false},
1283                 })
1284                 os.RemoveAll(cr.ArvMountPoint)
1285                 cr.CleanupDirs()
1286                 checkEmpty()
1287         }
1288
1289         for _, test := range []struct {
1290                 in  interface{}
1291                 out string
1292         }{
1293                 {in: "foo", out: `"foo"`},
1294                 {in: nil, out: `null`},
1295                 {in: map[string]int64{"foo": 123456789123456789}, out: `{"foo":123456789123456789}`},
1296         } {
1297                 i = 0
1298                 cr.ArvMountPoint = ""
1299                 cr.Container.Mounts = map[string]arvados.Mount{
1300                         "/mnt/test.json": {Kind: "json", Content: test.in},
1301                 }
1302                 bindmounts, err := cr.SetupMounts()
1303                 c.Check(err, IsNil)
1304                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1305                         "/mnt/test.json": {realTemp + "/json2/mountdata.json", true},
1306                 })
1307                 content, err := ioutil.ReadFile(realTemp + "/json2/mountdata.json")
1308                 c.Check(err, IsNil)
1309                 c.Check(content, DeepEquals, []byte(test.out))
1310                 os.RemoveAll(cr.ArvMountPoint)
1311                 cr.CleanupDirs()
1312                 checkEmpty()
1313         }
1314
1315         for _, test := range []struct {
1316                 in  interface{}
1317                 out string
1318         }{
1319                 {in: "foo", out: `foo`},
1320                 {in: nil, out: "error"},
1321                 {in: map[string]int64{"foo": 123456789123456789}, out: "error"},
1322         } {
1323                 i = 0
1324                 cr.ArvMountPoint = ""
1325                 cr.Container.Mounts = map[string]arvados.Mount{
1326                         "/mnt/test.txt": {Kind: "text", Content: test.in},
1327                 }
1328                 bindmounts, err := cr.SetupMounts()
1329                 if test.out == "error" {
1330                         c.Check(err.Error(), Equals, "content for mount \"/mnt/test.txt\" must be a string")
1331                 } else {
1332                         c.Check(err, IsNil)
1333                         c.Check(bindmounts, DeepEquals, map[string]bindmount{
1334                                 "/mnt/test.txt": {realTemp + "/text2/mountdata.text", true},
1335                         })
1336                         content, err := ioutil.ReadFile(realTemp + "/text2/mountdata.text")
1337                         c.Check(err, IsNil)
1338                         c.Check(content, DeepEquals, []byte(test.out))
1339                 }
1340                 os.RemoveAll(cr.ArvMountPoint)
1341                 cr.CleanupDirs()
1342                 checkEmpty()
1343         }
1344
1345         // Read-only mount points are allowed underneath output_dir mount point
1346         {
1347                 i = 0
1348                 cr.ArvMountPoint = ""
1349                 cr.Container.Mounts = make(map[string]arvados.Mount)
1350                 cr.Container.Mounts = map[string]arvados.Mount{
1351                         "/tmp":     {Kind: "tmp"},
1352                         "/tmp/foo": {Kind: "collection"},
1353                 }
1354                 cr.Container.OutputPath = "/tmp"
1355
1356                 os.MkdirAll(realTemp+"/keep1/tmp0", os.ModePerm)
1357
1358                 bindmounts, err := cr.SetupMounts()
1359                 c.Check(err, IsNil)
1360                 c.Check(am.Cmd, DeepEquals, []string{"arv-mount", "--foreground",
1361                         "--read-write", "--storage-classes", "default", "--crunchstat-interval=5",
1362                         "--file-cache", "512", "--mount-tmp", "tmp0", "--mount-by-pdh", "by_id", "--disable-event-listening", "--mount-by-id", "by_uuid", realTemp + "/keep1"})
1363                 c.Check(bindmounts, DeepEquals, map[string]bindmount{
1364                         "/tmp":     {realTemp + "/tmp2", false},
1365                         "/tmp/foo": {realTemp + "/keep1/tmp0", true},
1366                 })
1367                 os.RemoveAll(cr.ArvMountPoint)
1368                 cr.CleanupDirs()
1369                 checkEmpty()
1370         }
1371
1372         // Writable mount points copied to output_dir mount point
1373         {
1374                 i = 0
1375                 cr.ArvMountPoint = ""
1376                 cr.Container.Mounts = make(map[string]arvados.Mount)
1377                 cr.Container.Mounts = map[string]arvados.Mount{
1378                         "/tmp": {Kind: "tmp"},
1379                         "/tmp/foo": {Kind: "collection",
1380                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541c+53",
1381                                 Writable:         true},
1382                         "/tmp/bar": {Kind: "collection",
1383                                 PortableDataHash: "59389a8f9ee9d399be35462a0f92541d+53",
1384                                 Path:             "baz",
1385                                 Writable:         true},
1386                 }
1387                 cr.Container.OutputPath = "/tmp"
1388
1389                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541c+53", os.ModePerm)
1390                 os.MkdirAll(realTemp+"/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz", os.ModePerm)
1391
1392                 rf, _ := os.Create(realTemp + "/keep1/by_id/59389a8f9ee9d399be35462a0f92541d+53/baz/quux")
1393                 rf.Write([]byte("bar"))
1394                 rf.Close()
1395
1396                 _, err := cr.SetupMounts()
1397                 c.Check(err, IsNil)
1398                 _, err = os.Stat(cr.HostOutputDir + "/foo")
1399                 c.Check(err, IsNil)
1400                 _, err = os.Stat(cr.HostOutputDir + "/bar/quux")
1401                 c.Check(err, IsNil)
1402                 os.RemoveAll(cr.ArvMountPoint)
1403                 cr.CleanupDirs()
1404                 checkEmpty()
1405         }
1406
1407         // Only mount points of kind 'collection' are allowed underneath output_dir mount point
1408         {
1409                 i = 0
1410                 cr.ArvMountPoint = ""
1411                 cr.Container.Mounts = make(map[string]arvados.Mount)
1412                 cr.Container.Mounts = map[string]arvados.Mount{
1413                         "/tmp":     {Kind: "tmp"},
1414                         "/tmp/foo": {Kind: "tmp"},
1415                 }
1416                 cr.Container.OutputPath = "/tmp"
1417
1418                 _, err := cr.SetupMounts()
1419                 c.Check(err, NotNil)
1420                 c.Check(err, ErrorMatches, `only mount points of kind 'collection', 'text' or 'json' are supported underneath the output_path.*`)
1421                 os.RemoveAll(cr.ArvMountPoint)
1422                 cr.CleanupDirs()
1423                 checkEmpty()
1424         }
1425
1426         // Only mount point of kind 'collection' is allowed for stdin
1427         {
1428                 i = 0
1429                 cr.ArvMountPoint = ""
1430                 cr.Container.Mounts = make(map[string]arvados.Mount)
1431                 cr.Container.Mounts = map[string]arvados.Mount{
1432                         "stdin": {Kind: "tmp"},
1433                 }
1434
1435                 _, err := cr.SetupMounts()
1436                 c.Check(err, NotNil)
1437                 c.Check(err, ErrorMatches, `unsupported mount kind 'tmp' for stdin.*`)
1438                 os.RemoveAll(cr.ArvMountPoint)
1439                 cr.CleanupDirs()
1440                 checkEmpty()
1441         }
1442
1443         // git_tree mounts
1444         {
1445                 i = 0
1446                 cr.ArvMountPoint = ""
1447                 (*GitMountSuite)(nil).useTestGitServer(c)
1448                 cr.token = arvadostest.ActiveToken
1449                 cr.Container.Mounts = make(map[string]arvados.Mount)
1450                 cr.Container.Mounts = map[string]arvados.Mount{
1451                         "/tip": {
1452                                 Kind:   "git_tree",
1453                                 UUID:   arvadostest.Repository2UUID,
1454                                 Commit: "fd3531f42995344f36c30b79f55f27b502f3d344",
1455                                 Path:   "/",
1456                         },
1457                         "/non-tip": {
1458                                 Kind:   "git_tree",
1459                                 UUID:   arvadostest.Repository2UUID,
1460                                 Commit: "5ebfab0522851df01fec11ec55a6d0f4877b542e",
1461                                 Path:   "/",
1462                         },
1463                 }
1464                 cr.Container.OutputPath = "/tmp"
1465
1466                 bindmounts, err := cr.SetupMounts()
1467                 c.Check(err, IsNil)
1468
1469                 for path, mount := range bindmounts {
1470                         c.Check(mount.ReadOnly, Equals, !cr.Container.Mounts[path].Writable, Commentf("%s %#v", path, mount))
1471                 }
1472
1473                 data, err := ioutil.ReadFile(bindmounts["/tip"].HostPath + "/dir1/dir2/file with mode 0644")
1474                 c.Check(err, IsNil)
1475                 c.Check(string(data), Equals, "\000\001\002\003")
1476                 _, err = ioutil.ReadFile(bindmounts["/tip"].HostPath + "/file only on testbranch")
1477                 c.Check(err, FitsTypeOf, &os.PathError{})
1478                 c.Check(os.IsNotExist(err), Equals, true)
1479
1480                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/dir1/dir2/file with mode 0644")
1481                 c.Check(err, IsNil)
1482                 c.Check(string(data), Equals, "\000\001\002\003")
1483                 data, err = ioutil.ReadFile(bindmounts["/non-tip"].HostPath + "/file only on testbranch")
1484                 c.Check(err, IsNil)
1485                 c.Check(string(data), Equals, "testfile\n")
1486
1487                 cr.CleanupDirs()
1488                 checkEmpty()
1489         }
1490 }
1491
1492 func (s *TestSuite) TestStdout(c *C) {
1493         helperRecord := `{
1494                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1495                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1496                 "cwd": "/bin",
1497                 "environment": {"FROBIZ": "bilbo"},
1498                 "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"} },
1499                 "output_path": "/tmp",
1500                 "priority": 1,
1501                 "runtime_constraints": {},
1502                 "state": "Locked"
1503         }`
1504
1505         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1506                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1507         })
1508
1509         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1510         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1511         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1512 }
1513
1514 // Used by the TestStdoutWithWrongPath*()
1515 func (s *TestSuite) stdoutErrorRunHelper(c *C, record string, fn func()) (*ArvTestClient, *ContainerRunner, error) {
1516         err := json.Unmarshal([]byte(record), &s.api.Container)
1517         c.Assert(err, IsNil)
1518         s.executor.runFunc = fn
1519         s.runner.RunArvMount = (&ArvMountCmdLine{}).ArvMountTest
1520         s.runner.MkArvClient = func(token string) (IArvadosClient, IKeepClient, *arvados.Client, error) {
1521                 return s.api, &KeepTestClient{}, nil, nil
1522         }
1523         return s.api, s.runner, s.runner.Run()
1524 }
1525
1526 func (s *TestSuite) TestStdoutWithWrongPath(c *C) {
1527         _, _, err := s.stdoutErrorRunHelper(c, `{
1528     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "file", "path":"/tmpa.out"} },
1529     "output_path": "/tmp",
1530     "state": "Locked"
1531 }`, func() {})
1532         c.Check(err, ErrorMatches, ".*Stdout path does not start with OutputPath.*")
1533 }
1534
1535 func (s *TestSuite) TestStdoutWithWrongKindTmp(c *C) {
1536         _, _, err := s.stdoutErrorRunHelper(c, `{
1537     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "tmp", "path":"/tmp/a.out"} },
1538     "output_path": "/tmp",
1539     "state": "Locked"
1540 }`, func() {})
1541         c.Check(err, ErrorMatches, ".*unsupported mount kind 'tmp' for stdout.*")
1542 }
1543
1544 func (s *TestSuite) TestStdoutWithWrongKindCollection(c *C) {
1545         _, _, err := s.stdoutErrorRunHelper(c, `{
1546     "mounts": {"/tmp": {"kind": "tmp"}, "stdout": {"kind": "collection", "path":"/tmp/a.out"} },
1547     "output_path": "/tmp",
1548     "state": "Locked"
1549 }`, func() {})
1550         c.Check(err, ErrorMatches, ".*unsupported mount kind 'collection' for stdout.*")
1551 }
1552
1553 func (s *TestSuite) TestFullRunWithAPI(c *C) {
1554         s.fullRunHelper(c, `{
1555     "command": ["/bin/sh", "-c", "true $ARVADOS_API_HOST"],
1556     "container_image": "`+arvadostest.DockerImage112PDH+`",
1557     "cwd": "/bin",
1558     "environment": {},
1559     "mounts": {"/tmp": {"kind": "tmp"} },
1560     "output_path": "/tmp",
1561     "priority": 1,
1562     "runtime_constraints": {"API": true},
1563     "state": "Locked"
1564 }`, nil, 0, func() {
1565                 c.Check(s.executor.created.Env["ARVADOS_API_HOST"], Equals, os.Getenv("ARVADOS_API_HOST"))
1566                 s.executor.exit <- 3
1567         })
1568         c.Check(s.api.CalledWith("container.exit_code", 3), NotNil)
1569         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1570         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*status code 3\n.*`)
1571 }
1572
1573 func (s *TestSuite) TestFullRunSetOutput(c *C) {
1574         defer os.Setenv("ARVADOS_API_HOST", os.Getenv("ARVADOS_API_HOST"))
1575         os.Setenv("ARVADOS_API_HOST", "test.arvados.org")
1576         s.fullRunHelper(c, `{
1577     "command": ["/bin/sh", "-c", "echo $ARVADOS_API_HOST"],
1578     "container_image": "`+arvadostest.DockerImage112PDH+`",
1579     "cwd": "/bin",
1580     "environment": {},
1581     "mounts": {"/tmp": {"kind": "tmp"} },
1582     "output_path": "/tmp",
1583     "priority": 1,
1584     "runtime_constraints": {"API": true},
1585     "state": "Locked"
1586 }`, nil, 0, func() {
1587                 s.api.Container.Output = arvadostest.DockerImage112PDH
1588         })
1589
1590         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1591         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1592         c.Check(s.api.CalledWith("container.output", arvadostest.DockerImage112PDH), NotNil)
1593 }
1594
1595 func (s *TestSuite) TestArvMountRuntimeStatusWarning(c *C) {
1596         s.runner.RunArvMount = func([]string, string) (*exec.Cmd, error) {
1597                 os.Mkdir(s.runner.ArvMountPoint+"/by_id", 0666)
1598                 ioutil.WriteFile(s.runner.ArvMountPoint+"/by_id/README", nil, 0666)
1599                 return s.runner.ArvMountCmd([]string{"bash", "-c", "echo >&2 Test: Keep write error: I am a teapot; sleep 3"}, "")
1600         }
1601         s.executor.runFunc = func() {
1602                 time.Sleep(time.Second)
1603                 s.executor.exit <- 137
1604         }
1605         record := `{
1606     "command": ["sleep", "1"],
1607     "container_image": "` + arvadostest.DockerImage112PDH + `",
1608     "cwd": "/bin",
1609     "environment": {},
1610     "mounts": {"/tmp": {"kind": "tmp"} },
1611     "output_path": "/tmp",
1612     "priority": 1,
1613     "runtime_constraints": {"API": true},
1614     "state": "Locked"
1615 }`
1616         err := json.Unmarshal([]byte(record), &s.api.Container)
1617         c.Assert(err, IsNil)
1618         err = s.runner.Run()
1619         c.Assert(err, IsNil)
1620         c.Check(s.api.CalledWith("container.exit_code", 137), NotNil)
1621         c.Check(s.api.CalledWith("container.runtime_status.warning", "arv-mount: Keep write error"), NotNil)
1622         c.Check(s.api.CalledWith("container.runtime_status.warningDetail", "Test: Keep write error: I am a teapot"), NotNil)
1623         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1624         c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Container exited with status code 137 \(signal 9, SIGKILL\).*`)
1625 }
1626
1627 func (s *TestSuite) TestStdoutWithExcludeFromOutputMountPointUnderOutputDir(c *C) {
1628         helperRecord := `{
1629                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1630                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1631                 "cwd": "/bin",
1632                 "environment": {"FROBIZ": "bilbo"},
1633                 "mounts": {
1634         "/tmp": {"kind": "tmp"},
1635         "/tmp/foo": {"kind": "collection",
1636                      "portable_data_hash": "a3e8f74c6f101eae01fa08bfb4e49b3a+54",
1637                      "exclude_from_output": true
1638         },
1639         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1640     },
1641                 "output_path": "/tmp",
1642                 "priority": 1,
1643                 "runtime_constraints": {},
1644                 "state": "Locked"
1645         }`
1646
1647         extraMounts := []string{"a3e8f74c6f101eae01fa08bfb4e49b3a+54"}
1648
1649         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1650                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1651         })
1652
1653         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1654         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1655         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out\n"), NotNil)
1656 }
1657
1658 func (s *TestSuite) TestStdoutWithMultipleMountPointsUnderOutputDir(c *C) {
1659         helperRecord := `{
1660                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1661                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1662                 "cwd": "/bin",
1663                 "environment": {"FROBIZ": "bilbo"},
1664                 "mounts": {
1665         "/tmp": {"kind": "tmp"},
1666         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/file2_in_main.txt"},
1667         "/tmp/foo/sub1": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1"},
1668         "/tmp/foo/sub1file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/file2_in_subdir1.txt"},
1669         "/tmp/foo/baz/sub2file2": {"kind": "collection", "portable_data_hash": "a0def87f80dd594d4675809e83bd4f15+367", "path":"/subdir1/subdir2/file2_in_subdir2.txt"},
1670         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1671     },
1672                 "output_path": "/tmp",
1673                 "priority": 1,
1674                 "runtime_constraints": {},
1675                 "state": "Locked"
1676         }`
1677
1678         extraMounts := []string{
1679                 "a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt",
1680                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1681                 "a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt",
1682         }
1683
1684         api, _, realtemp := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1685                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1686         })
1687
1688         c.Check(s.executor.created.BindMounts, DeepEquals, map[string]bindmount{
1689                 "/tmp":                   {realtemp + "/tmp1", false},
1690                 "/tmp/foo/bar":           {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/file2_in_main.txt", true},
1691                 "/tmp/foo/baz/sub2file2": {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/subdir2/file2_in_subdir2.txt", true},
1692                 "/tmp/foo/sub1":          {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1", true},
1693                 "/tmp/foo/sub1file2":     {s.keepmount + "/by_id/a0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt", true},
1694         })
1695
1696         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1697         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1698         for _, v := range api.Content {
1699                 if v["collection"] != nil {
1700                         c.Check(v["ensure_unique_name"], Equals, true)
1701                         collection := v["collection"].(arvadosclient.Dict)
1702                         if strings.Index(collection["name"].(string), "output") == 0 {
1703                                 manifest := collection["manifest_text"].(string)
1704
1705                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1706 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 9:18:bar 36:18:sub1file2
1707 ./foo/baz 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 9:18:sub2file2
1708 ./foo/sub1 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396cabcdefghij6419876543234@569fa8c4 0:9:file1_in_subdir1.txt 9:18:file2_in_subdir1.txt
1709 ./foo/sub1/subdir2 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0bcdefghijk544332211@569fa8c5 0:9:file1_in_subdir2.txt 9:18:file2_in_subdir2.txt
1710 `)
1711                         }
1712                 }
1713         }
1714 }
1715
1716 func (s *TestSuite) TestStdoutWithMountPointsUnderOutputDirDenormalizedManifest(c *C) {
1717         helperRecord := `{
1718                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1719                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1720                 "cwd": "/bin",
1721                 "environment": {"FROBIZ": "bilbo"},
1722                 "mounts": {
1723         "/tmp": {"kind": "tmp"},
1724         "/tmp/foo/bar": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/subdir1/file2_in_subdir1.txt"},
1725         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1726     },
1727                 "output_path": "/tmp",
1728                 "priority": 1,
1729                 "runtime_constraints": {},
1730                 "state": "Locked"
1731         }`
1732
1733         extraMounts := []string{
1734                 "b0def87f80dd594d4675809e83bd4f15+367/subdir1/file2_in_subdir1.txt",
1735         }
1736
1737         s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1738                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1739         })
1740
1741         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1742         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1743         for _, v := range s.api.Content {
1744                 if v["collection"] != nil {
1745                         collection := v["collection"].(arvadosclient.Dict)
1746                         if strings.Index(collection["name"].(string), "output") == 0 {
1747                                 manifest := collection["manifest_text"].(string)
1748
1749                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1750 ./foo 3e426d509afffb85e06c4c96a7c15e91+27+Aa124ac75e5168396c73c0abcdefgh11234567890@569fa8c3 10:17:bar
1751 `)
1752                         }
1753                 }
1754         }
1755 }
1756
1757 func (s *TestSuite) TestOutputError(c *C) {
1758         helperRecord := `{
1759                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1760                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1761                 "cwd": "/bin",
1762                 "environment": {"FROBIZ": "bilbo"},
1763                 "mounts": {
1764                         "/tmp": {"kind": "tmp"}
1765                 },
1766                 "output_path": "/tmp",
1767                 "priority": 1,
1768                 "runtime_constraints": {},
1769                 "state": "Locked"
1770         }`
1771         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1772                 os.Symlink("/etc/hosts", s.runner.HostOutputDir+"/baz")
1773         })
1774
1775         c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1776 }
1777
1778 func (s *TestSuite) TestStdinCollectionMountPoint(c *C) {
1779         helperRecord := `{
1780                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1781                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1782                 "cwd": "/bin",
1783                 "environment": {"FROBIZ": "bilbo"},
1784                 "mounts": {
1785         "/tmp": {"kind": "tmp"},
1786         "stdin": {"kind": "collection", "portable_data_hash": "b0def87f80dd594d4675809e83bd4f15+367", "path": "/file1_in_main.txt"},
1787         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1788     },
1789                 "output_path": "/tmp",
1790                 "priority": 1,
1791                 "runtime_constraints": {},
1792                 "state": "Locked"
1793         }`
1794
1795         extraMounts := []string{
1796                 "b0def87f80dd594d4675809e83bd4f15+367/file1_in_main.txt",
1797         }
1798
1799         api, _, _ := s.fullRunHelper(c, helperRecord, extraMounts, 0, func() {
1800                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1801         })
1802
1803         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1804         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1805         for _, v := range api.Content {
1806                 if v["collection"] != nil {
1807                         collection := v["collection"].(arvadosclient.Dict)
1808                         if strings.Index(collection["name"].(string), "output") == 0 {
1809                                 manifest := collection["manifest_text"].(string)
1810                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1811 `)
1812                         }
1813                 }
1814         }
1815 }
1816
1817 func (s *TestSuite) TestStdinJsonMountPoint(c *C) {
1818         helperRecord := `{
1819                 "command": ["/bin/sh", "-c", "echo $FROBIZ"],
1820                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1821                 "cwd": "/bin",
1822                 "environment": {"FROBIZ": "bilbo"},
1823                 "mounts": {
1824         "/tmp": {"kind": "tmp"},
1825         "stdin": {"kind": "json", "content": "foo"},
1826         "stdout": {"kind": "file", "path": "/tmp/a/b/c.out"}
1827     },
1828                 "output_path": "/tmp",
1829                 "priority": 1,
1830                 "runtime_constraints": {},
1831                 "state": "Locked"
1832         }`
1833
1834         api, _, _ := s.fullRunHelper(c, helperRecord, nil, 0, func() {
1835                 fmt.Fprintln(s.executor.created.Stdout, s.executor.created.Env["FROBIZ"])
1836         })
1837
1838         c.Check(api.CalledWith("container.exit_code", 0), NotNil)
1839         c.Check(api.CalledWith("container.state", "Complete"), NotNil)
1840         for _, v := range api.Content {
1841                 if v["collection"] != nil {
1842                         collection := v["collection"].(arvadosclient.Dict)
1843                         if strings.Index(collection["name"].(string), "output") == 0 {
1844                                 manifest := collection["manifest_text"].(string)
1845                                 c.Check(manifest, Equals, `./a/b 307372fa8fd5c146b22ae7a45b49bc31+6 0:6:c.out
1846 `)
1847                         }
1848                 }
1849         }
1850 }
1851
1852 func (s *TestSuite) TestStderrMount(c *C) {
1853         api, cr, _ := s.fullRunHelper(c, `{
1854     "command": ["/bin/sh", "-c", "echo hello;exit 1"],
1855     "container_image": "`+arvadostest.DockerImage112PDH+`",
1856     "cwd": ".",
1857     "environment": {},
1858     "mounts": {"/tmp": {"kind": "tmp"},
1859                "stdout": {"kind": "file", "path": "/tmp/a/out.txt"},
1860                "stderr": {"kind": "file", "path": "/tmp/b/err.txt"}},
1861     "output_path": "/tmp",
1862     "priority": 1,
1863     "runtime_constraints": {},
1864     "state": "Locked"
1865 }`, nil, 1, func() {
1866                 fmt.Fprintln(s.executor.created.Stdout, "hello")
1867                 fmt.Fprintln(s.executor.created.Stderr, "oops")
1868         })
1869
1870         final := api.CalledWith("container.state", "Complete")
1871         c.Assert(final, NotNil)
1872         c.Check(final["container"].(arvadosclient.Dict)["exit_code"], Equals, 1)
1873         c.Check(final["container"].(arvadosclient.Dict)["log"], NotNil)
1874
1875         c.Check(cr.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", "./a b1946ac92492d2347c6235b4d2611184+6 0:6:out.txt\n./b 38af5c54926b620264ab1501150cf189+5 0:5:err.txt\n"), NotNil)
1876 }
1877
1878 func (s *TestSuite) TestNumberRoundTrip(c *C) {
1879         s.api.callraw = true
1880         err := s.runner.fetchContainerRecord()
1881         c.Assert(err, IsNil)
1882         jsondata, err := json.Marshal(s.runner.Container.Mounts["/json"].Content)
1883         c.Logf("%#v", s.runner.Container)
1884         c.Check(err, IsNil)
1885         c.Check(string(jsondata), Equals, `{"number":123456789123456789}`)
1886 }
1887
1888 func (s *TestSuite) TestFullBrokenDocker(c *C) {
1889         nextState := ""
1890         for _, setup := range []func(){
1891                 func() {
1892                         c.Log("// waitErr = ocl runtime error")
1893                         s.executor.waitErr = errors.New(`Error response from daemon: oci runtime error: container_linux.go:247: starting container process caused "process_linux.go:359: container init caused \"rootfs_linux.go:54: mounting \\\"/tmp/keep453790790/by_id/99999999999999999999999999999999+99999/myGenome\\\" to rootfs \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged\\\" at \\\"/tmp/docker/overlay2/9999999999999999999999999999999999999999999999999999999999999999/merged/keep/99999999999999999999999999999999+99999/myGenome\\\" caused \\\"no such file or directory\\\"\""`)
1894                         nextState = "Cancelled"
1895                 },
1896                 func() {
1897                         c.Log("// loadErr = cannot connect")
1898                         s.executor.loadErr = errors.New("Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?")
1899                         *brokenNodeHook = c.MkDir() + "/broken-node-hook"
1900                         err := ioutil.WriteFile(*brokenNodeHook, []byte("#!/bin/sh\nexec echo killme\n"), 0700)
1901                         c.Assert(err, IsNil)
1902                         nextState = "Queued"
1903                 },
1904         } {
1905                 s.SetUpTest(c)
1906                 setup()
1907                 s.fullRunHelper(c, `{
1908     "command": ["echo", "hello world"],
1909     "container_image": "`+arvadostest.DockerImage112PDH+`",
1910     "cwd": ".",
1911     "environment": {},
1912     "mounts": {"/tmp": {"kind": "tmp"} },
1913     "output_path": "/tmp",
1914     "priority": 1,
1915     "runtime_constraints": {},
1916     "state": "Locked"
1917 }`, nil, 0, func() {})
1918                 c.Check(s.api.CalledWith("container.state", nextState), NotNil)
1919                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*unable to run containers.*")
1920                 if *brokenNodeHook != "" {
1921                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Running broken node hook.*")
1922                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*killme.*")
1923                         c.Check(s.api.Logs["crunch-run"].String(), Not(Matches), "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1924                 } else {
1925                         c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Writing /var/lock/crunch-run-broken to mark node as broken.*")
1926                 }
1927         }
1928 }
1929
1930 func (s *TestSuite) TestBadCommand(c *C) {
1931         for _, startError := range []string{
1932                 `panic: standard_init_linux.go:175: exec user process caused "no such file or directory"`,
1933                 `Error response from daemon: Cannot start container 41f26cbc43bcc1280f4323efb1830a394ba8660c9d1c2b564ba42bf7f7694845: [8] System error: no such file or directory`,
1934                 `Error response from daemon: Cannot start container 58099cd76c834f3dc2a4fb76c8028f049ae6d4fdf0ec373e1f2cfea030670c2d: [8] System error: exec: "foobar": executable file not found in $PATH`,
1935         } {
1936                 s.SetUpTest(c)
1937                 s.executor.startErr = errors.New(startError)
1938                 s.fullRunHelper(c, `{
1939     "command": ["echo", "hello world"],
1940     "container_image": "`+arvadostest.DockerImage112PDH+`",
1941     "cwd": ".",
1942     "environment": {},
1943     "mounts": {"/tmp": {"kind": "tmp"} },
1944     "output_path": "/tmp",
1945     "priority": 1,
1946     "runtime_constraints": {},
1947     "state": "Locked"
1948 }`, nil, 0, func() {})
1949                 c.Check(s.api.CalledWith("container.state", "Cancelled"), NotNil)
1950                 c.Check(s.api.Logs["crunch-run"].String(), Matches, "(?ms).*Possible causes:.*is missing.*")
1951         }
1952 }
1953
1954 func (s *TestSuite) TestSecretTextMountPoint(c *C) {
1955         helperRecord := `{
1956                 "command": ["true"],
1957                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1958                 "cwd": "/bin",
1959                 "mounts": {
1960                     "/tmp": {"kind": "tmp"},
1961                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1962                 },
1963                 "secret_mounts": {
1964                 },
1965                 "output_path": "/tmp",
1966                 "priority": 1,
1967                 "runtime_constraints": {},
1968                 "state": "Locked"
1969         }`
1970
1971         s.fullRunHelper(c, helperRecord, nil, 0, func() {
1972                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
1973                 c.Check(err, IsNil)
1974                 c.Check(string(content), Equals, "mypassword")
1975         })
1976
1977         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
1978         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
1979         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), NotNil)
1980         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), IsNil)
1981
1982         // under secret mounts, not captured in output
1983         helperRecord = `{
1984                 "command": ["true"],
1985                 "container_image": "` + arvadostest.DockerImage112PDH + `",
1986                 "cwd": "/bin",
1987                 "mounts": {
1988                     "/tmp": {"kind": "tmp"}
1989                 },
1990                 "secret_mounts": {
1991                     "/tmp/secret.conf": {"kind": "text", "content": "mypassword"}
1992                 },
1993                 "output_path": "/tmp",
1994                 "priority": 1,
1995                 "runtime_constraints": {},
1996                 "state": "Locked"
1997         }`
1998
1999         s.SetUpTest(c)
2000         s.fullRunHelper(c, helperRecord, nil, 0, func() {
2001                 content, err := ioutil.ReadFile(s.runner.HostOutputDir + "/secret.conf")
2002                 c.Check(err, IsNil)
2003                 c.Check(string(content), Equals, "mypassword")
2004         })
2005
2006         c.Check(s.api.CalledWith("container.exit_code", 0), NotNil)
2007         c.Check(s.api.CalledWith("container.state", "Complete"), NotNil)
2008         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ". 34819d7beeabb9260a5c854bc85b3e44+10 0:10:secret.conf\n"), IsNil)
2009         c.Check(s.runner.ContainerArvClient.(*ArvTestClient).CalledWith("collection.manifest_text", ""), NotNil)
2010 }
2011
2012 type FakeProcess struct {
2013         cmdLine []string
2014 }
2015
2016 func (fp FakeProcess) CmdlineSlice() ([]string, error) {
2017         return fp.cmdLine, nil
2018 }