2755: Merge branch '2755-require-keep-permission' refs #2755

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index e30efd902445fe46eebd467800248428213c45ef..8b4717734a0450f2f25d652eeb35589e4fa2ebc9 100755 (executable)
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -641,8 +641,8 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
     $command .= "&& exec arv-mount --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
     if ($docker_image)
     {
-      $command .= "crunchstat -cgroup-parent=/sys/fs/cgroup/lxc -cgroup-cid=$ENV{TASK_WORK}/docker.cid -poll=1000 ";
-      $command .= "$docker_bin run -i -a stdin -a stdout -a stderr -cidfile=$ENV{TASK_WORK}/docker.cid ";
+      $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -cgroup-parent=docker -cgroup-cid=$ENV{TASK_WORK}/docker.cid -poll=10000 ";
+      $command .= "$docker_bin run -i -a stdin -a stdout -a stderr --cidfile=$ENV{TASK_WORK}/docker.cid ";
       # Dynamically configure the container to use the host system as its
       # DNS server.  Get the host's global addresses from the ip command,
       # and turn them into docker --dns options using gawk.
@@ -661,8 +661,9 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
       }
       $command .= "\Q$docker_image\E ";
     } else {
-      $command .= "crunchstat -cgroup-path=/sys/fs/cgroup "
+      $command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 "
     }
+    $command .= "stdbuf -o0 -e0 ";
     $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
     my @execargs = ('bash', '-c', $command);
     srun (\@srunargs, \@execargs, undef, $build_script_to_send);

diff --git a/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb b/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb
index 4a2bafde04eda0dc6c3639d963796a1d95734c8a..76a228d9d580c21d3483e9a01643255d67c012c4 100644 (file)
--- a/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb
+++ b/services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb
@@ -21,6 +21,12 @@ class Arvados::V1::ApiClientAuthorizationsController < ApplicationController
   end
 
   def create
+    # Note: the user could specify a owner_uuid for a different user, which on
+    # the surface appears to be a security hole.  However, the record will be
+    # rejected before being saved to the database by the ApiClientAuthorization
+    # model which enforces that user_id == current user or the user is an
+    # admin.
+
     if resource_attrs[:owner_uuid]
       # The model has an owner_id attribute instead of owner_uuid, but
       # we can't expect the client to know the local numeric ID. We

diff --git a/services/crunch/crunchstat/src/arvados.org/crunchstat/crunchstat.go b/services/crunch/crunchstat/src/arvados.org/crunchstat/crunchstat.go
index 75284856fad62e102433a92282fb3b17f52908f7..f8d27ec01b79d7f25d2405fe2fb12085fce17ef2 100644 (file)
--- a/services/crunch/crunchstat/src/arvados.org/crunchstat/crunchstat.go
+++ b/services/crunch/crunchstat/src/arvados.org/crunchstat/crunchstat.go
@@ -42,7 +42,28 @@ func OutputChannel(stdout chan string, stderr chan string) {
        }
 }
 
-func PollCgroupStats(cgroup_path string, stderr chan string, poll int64) {
+func FindStat(cgroup_root string, cgroup_parent string, container_id string, statgroup string, stat string) string {
+       var path string
+       path = fmt.Sprintf("%s/%s/%s/%s/%s.%s", cgroup_root, statgroup, cgroup_parent, container_id, statgroup, stat)
+       if _, err := os.Stat(path); err == nil {
+               return path
+       }
+       path = fmt.Sprintf("%s/%s/%s/%s.%s", cgroup_root, cgroup_parent, container_id, statgroup, stat)
+       if _, err := os.Stat(path); err == nil {
+               return path
+       }
+       path = fmt.Sprintf("%s/%s/%s.%s", cgroup_root, statgroup, statgroup, stat)
+       if _, err := os.Stat(path); err == nil {
+               return path
+       }
+       path = fmt.Sprintf("%s/%s.%s", cgroup_root, statgroup, stat)
+       if _, err := os.Stat(path); err == nil {
+               return path
+       }
+       return ""
+}
+
+func PollCgroupStats(cgroup_root string, cgroup_parent string, container_id string, stderr chan string, poll int64) {
        //var last_usage int64 = 0
        var last_user int64 = 0
        var last_sys int64 = 0
@@ -57,11 +78,24 @@ func PollCgroupStats(cgroup_path string, stderr chan string, poll int64) {
 
        disk := make(map[string]*Disk)
 
-       //cpuacct_usage := fmt.Sprintf("%s/cpuacct.usage", cgroup_path)
-       cpuacct_stat := fmt.Sprintf("%s/cpuacct.stat", cgroup_path)
-       blkio_io_service_bytes := fmt.Sprintf("%s/blkio.io_service_bytes", cgroup_path)
-       cpuset_cpus := fmt.Sprintf("%s/cpuset.cpus", cgroup_path)
-       memory_stat := fmt.Sprintf("%s/memory.stat", cgroup_path)
+       //cpuacct_usage := FindStat(cgroup_path, "cpuacct", "usage")
+       cpuacct_stat := FindStat(cgroup_root, cgroup_parent, container_id, "cpuacct", "stat")
+       blkio_io_service_bytes := FindStat(cgroup_root, cgroup_parent, container_id, "blkio", "io_service_bytes")
+       cpuset_cpus := FindStat(cgroup_root, cgroup_parent, container_id, "cpuset", "cpus")
+       memory_stat := FindStat(cgroup_root, cgroup_parent, container_id, "memory", "stat")
+
+       if cpuacct_stat != "" {
+               stderr <- fmt.Sprintf("crunchstat: reading stats from %s", cpuacct_stat)
+       }
+       if blkio_io_service_bytes != "" {
+               stderr <- fmt.Sprintf("crunchstat: reading stats from %s", blkio_io_service_bytes)
+       }
+       if cpuset_cpus != "" {
+               stderr <- fmt.Sprintf("crunchstat: reading stats from %s", cpuset_cpus)
+       }
+       if memory_stat != "" {
+               stderr <- fmt.Sprintf("crunchstat: reading stats from %s", memory_stat)
+       }
 
        var elapsed int64 = poll
 
@@ -79,7 +113,7 @@ func PollCgroupStats(cgroup_path string, stderr chan string, poll int64) {
                        c.Close()
                }*/
                var cpus int64 = 0
-               {
+               if cpuset_cpus != "" {
                        c, _ := os.Open(cpuset_cpus)
                        b, _ := ioutil.ReadAll(c)
                        sp := strings.Split(string(b), ",")
@@ -103,7 +137,7 @@ func PollCgroupStats(cgroup_path string, stderr chan string, poll int64) {
                if cpus == 0 {
                        cpus = 1
                }
-               {
+               if cpuacct_stat != "" {
                        c, _ := os.Open(cpuacct_stat)
                        b, _ := ioutil.ReadAll(c)
                        var next_user int64
@@ -135,7 +169,7 @@ func PollCgroupStats(cgroup_path string, stderr chan string, poll int64) {
                        last_user = next_user
                        last_sys = next_sys
                }
-               {
+               if blkio_io_service_bytes != "" {
                        c, _ := os.Open(blkio_io_service_bytes)
                        b := bufio.NewScanner(c)
                        var device, op string
@@ -164,7 +198,7 @@ func PollCgroupStats(cgroup_path string, stderr chan string, poll int64) {
                        c.Close()
                }
 
-               {
+               if memory_stat != "" {
                        c, _ := os.Open(memory_stat)
                        b := bufio.NewScanner(c)
                        var stat string
@@ -189,15 +223,15 @@ func PollCgroupStats(cgroup_path string, stderr chan string, poll int64) {
 func main() {
 
        var (
-               cgroup_path    string
+               cgroup_root    string
                cgroup_parent  string
                cgroup_cidfile string
                wait           int64
                poll           int64
        )
 
-       flag.StringVar(&cgroup_path, "cgroup-path", "", "Direct path to cgroup")
-       flag.StringVar(&cgroup_parent, "cgroup-parent", "", "Path to parent cgroup")
+       flag.StringVar(&cgroup_root, "cgroup-root", "", "Root of cgroup tree")
+       flag.StringVar(&cgroup_parent, "cgroup-parent", "", "Name of container parent under cgroup")
        flag.StringVar(&cgroup_cidfile, "cgroup-cid", "", "Path to container id file")
        flag.Int64Var(&wait, "wait", 5, "Maximum time (in seconds) to wait for cid file to show up")
        flag.Int64Var(&poll, "poll", 1000, "Polling frequency, in milliseconds")
@@ -206,8 +240,8 @@ func main() {
 
        logger := log.New(os.Stderr, "crunchstat: ", 0)
 
-       if cgroup_path == "" && cgroup_cidfile == "" {
-               logger.Fatal("Must provide either -cgroup-path or -cgroup-cid")
+       if cgroup_root == "" {
+               logger.Fatal("Must provide either -cgroup-root")
        }
 
        // Make output channel
@@ -260,6 +294,7 @@ func main() {
        }
 
        // Read the cid file
+       var container_id string
        if cgroup_cidfile != "" {
                // wait up to 'wait' seconds for the cid file to appear
                var i time.Duration
@@ -268,26 +303,19 @@ func main() {
                        if err == nil {
                                cid, err2 := ioutil.ReadAll(f)
                                if err2 == nil && len(cid) > 0 {
-                                       cgroup_path = string(cid)
+                                       container_id = string(cid)
                                        f.Close()
                                        break
                                }
                        }
                        time.Sleep(100 * time.Millisecond)
                }
-               if cgroup_path == "" {
+               if cgroup_root == "" {
                        logger.Printf("Could not read cid file %s", cgroup_cidfile)
                }
        }
 
-       // add the parent prefix
-       if cgroup_parent != "" {
-               cgroup_path = fmt.Sprintf("%s/%s", cgroup_parent, cgroup_path)
-       }
-
-       logger.Print("Using cgroup ", cgroup_path)
-
-       go PollCgroupStats(cgroup_path, stderr_chan, poll)
+       go PollCgroupStats(cgroup_root, cgroup_parent, container_id, stderr_chan, poll)
 
        // Wait for each of stdout and stderr to drain
        <-finish_chan