10218: Added calls to "df -i" to record free i-nodes. Combined stdout and stderr...

[arvados.git] / services / crunch-run / crunchrun.go

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 44d68ca0a3db725963a38a2bbd1b8cc36842a90f..e0d707a5a538b7800c8ef70abaf7b618d3c3f321 100644 (file)
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -33,6 +33,7 @@ type IArvadosClient interface {
        Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error
        Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) error
        Call(method, resourceType, uuid, action string, parameters arvadosclient.Dict, output interface{}) error
+       CallRaw(method string, resourceType string, uuid string, action string, parameters arvadosclient.Dict) (reader io.ReadCloser, err error)
        Discovery(key string) (interface{}, error)
 }
 
@@ -95,7 +96,6 @@ type ContainerRunner struct {
        ArvMountExit   chan error
        finalState     string
 
-       infoLogger   io.WriteCloser
        statLogger   io.WriteCloser
        statReporter *crunchstat.Reporter
        statInterval time.Duration
@@ -129,7 +129,6 @@ func (runner *ContainerRunner) SetupSignals() {
 
        go func(sig chan os.Signal) {
                <-sig
-               log.Print("signal handler calling runner.stop()")
                runner.stop()
                signal.Stop(sig)
        }(runner.SigChan)
@@ -534,17 +533,29 @@ func (runner *ContainerRunner) LogNodeInfo() (err error) {
                },
                infoCommand{
                        label: "Disk Space",
-                       cmd:   []string{"df", "-m"},
+                       cmd:   []string{"df", "-m", "/"},
+               },
+               infoCommand{
+                       label: "Disk Space",
+                       cmd:   []string{"df", "-m", os.TempDir()},
+               },
+               infoCommand{
+                       label: "Disk INodes",
+                       cmd:   []string{"df", "-i", "/"},
+               },
+               infoCommand{
+                       label: "Disk INodes",
+                       cmd:   []string{"df", "-i", os.TempDir()},
                },
        }
 
        // Run commands with informational output to be logged.
        var out []byte
        for _, command := range commands {
-               out, err = exec.Command(command.cmd[0], command.cmd[1:]...).Output()
+               out, err = exec.Command(command.cmd[0], command.cmd[1:]...).CombinedOutput()
                if err != nil {
-                       return fmt.Errorf("While running command '%s': %v",
-                               command.cmd[0], err)
+                       return fmt.Errorf("While running command %q: %v",
+                               command.cmd, err)
                }
                logger.Println(command.label)
                for _, line := range strings.Split(string(out), "\n") {
@@ -559,6 +570,42 @@ func (runner *ContainerRunner) LogNodeInfo() (err error) {
        return nil
 }
 
+// Get and save the raw JSON container record from the API server
+func (runner *ContainerRunner) LogContainerRecord() (err error) {
+       w := &ArvLogWriter{
+               runner.ArvClient,
+               runner.Container.UUID,
+               "container",
+               runner.LogCollection.Open("container.json"),
+       }
+       // Get Container record JSON from the API Server
+       reader, err := runner.ArvClient.CallRaw("GET", "containers", runner.Container.UUID, "", nil)
+       if err != nil {
+               return fmt.Errorf("While retrieving container record from the API server: %v", err)
+       }
+       // Read the API server response as []byte
+       json_bytes, err := ioutil.ReadAll(reader)
+       if err != nil {
+               return fmt.Errorf("While reading container record API server response: %v", err)
+       }
+       // Decode the JSON []byte
+       var cr map[string]interface{}
+       if err = json.Unmarshal(json_bytes, &cr); err != nil {
+               return fmt.Errorf("While decoding the container record JSON response: %v", err)
+       }
+       // Re-encode it using indentation to improve readability
+       enc := json.NewEncoder(w)
+       enc.SetIndent("", "    ")
+       if err = enc.Encode(cr); err != nil {
+               return fmt.Errorf("While logging the JSON container record: %v", err)
+       }
+       err = w.Close()
+       if err != nil {
+               return fmt.Errorf("While closing container.json log: %v", err)
+       }
+       return nil
+}
+
 // AttachLogs connects the docker container stdout and stderr logs to the
 // Arvados logger which logs to Keep and the API server logs table.
 func (runner *ContainerRunner) AttachStreams() (err error) {
@@ -1056,6 +1103,11 @@ func (runner *ContainerRunner) Run() (err error) {
        if err != nil {
                return
        }
+       // Save container.json record on log collection
+       err = runner.LogContainerRecord()
+       if err != nil {
+               return
+       }
 
        runner.StartCrunchstat()