DataManagerToken = "320mkve8qkswstz7ff61glpk3mhgghmg67wmic7elw4z41pke1"
ManagementToken = "jg3ajndnq63sywcd50gbs5dskdc9ckkysb0nsqmfz08nwf17nl"
ActiveUserUUID = "zzzzz-tpzed-xurymjxw79nv3jz"
+ FederatedActiveUserUUID = "zbbbb-tpzed-xurymjxw79nv3jz"
SpectatorUserUUID = "zzzzz-tpzed-l1s2piq4t4mps8r"
UserAgreementCollection = "zzzzz-4zz18-uukreo9rbgwsujr" // user_agreement_in_anonymously_accessible_project
FooCollection = "zzzzz-4zz18-fy296fx3hot09f7"
role: Computational biologist
getting_started_shown: 2015-03-26 12:34:56.789000000 Z
+federated_active:
+ owner_uuid: zzzzz-tpzed-000000000000000
+ uuid: zbbbb-tpzed-xurymjxw79nv3jz
+ email: zbbbb-active-user@arvados.local
+ first_name: Active
+ last_name: User
+ identity_url: https://active-user.openid.local
+ is_active: true
+ is_admin: false
+ username: federatedactive
+ prefs:
+ profile:
+ organization: example.com
+ role: Computational biologist
+ getting_started_shown: 2015-03-26 12:34:56.789000000 Z
+
project_viewer:
owner_uuid: zzzzz-tpzed-000000000000000
uuid: zzzzz-tpzed-projectviewer1a
ArvMountExit chan error
finalState string
- statLogger io.WriteCloser
- statReporter *crunchstat.Reporter
- statInterval time.Duration
- cgroupRoot string
+ statLogger io.WriteCloser
+ statReporter *crunchstat.Reporter
+ hoststatLogger io.WriteCloser
+ hoststatReporter *crunchstat.Reporter
+ statInterval time.Duration
+ cgroupRoot string
// What we expect the container's cgroup parent to be.
expectCgroupParent string
// What we tell docker to use as the container's cgroup
func (runner *ContainerRunner) ProcessDockerAttach(containerReader io.Reader) {
// Handle docker log protocol
// https://docs.docker.com/engine/reference/api/docker_remote_api_v1.15/#attach-to-a-container
+ defer close(runner.loggingDone)
header := make([]byte, 8)
- for {
- _, readerr := io.ReadAtLeast(containerReader, header, 8)
-
- if readerr == nil {
- readsize := int64(header[7]) | (int64(header[6]) << 8) | (int64(header[5]) << 16) | (int64(header[4]) << 24)
- if header[0] == 1 {
- // stdout
- _, readerr = io.CopyN(runner.Stdout, containerReader, readsize)
- } else {
- // stderr
- _, readerr = io.CopyN(runner.Stderr, containerReader, readsize)
+ var err error
+ for err == nil {
+ _, err = io.ReadAtLeast(containerReader, header, 8)
+ if err != nil {
+ if err == io.EOF {
+ err = nil
}
+ break
}
+ readsize := int64(header[7]) | (int64(header[6]) << 8) | (int64(header[5]) << 16) | (int64(header[4]) << 24)
+ if header[0] == 1 {
+ // stdout
+ _, err = io.CopyN(runner.Stdout, containerReader, readsize)
+ } else {
+ // stderr
+ _, err = io.CopyN(runner.Stderr, containerReader, readsize)
+ }
+ }
- if readerr != nil {
- if readerr != io.EOF {
- runner.CrunchLog.Printf("While reading docker logs: %v", readerr)
- }
-
- closeerr := runner.Stdout.Close()
- if closeerr != nil {
- runner.CrunchLog.Printf("While closing stdout logs: %v", closeerr)
- }
+ if err != nil {
+ runner.CrunchLog.Printf("error reading docker logs: %v", err)
+ }
- closeerr = runner.Stderr.Close()
- if closeerr != nil {
- runner.CrunchLog.Printf("While closing stderr logs: %v", closeerr)
- }
+ err = runner.Stdout.Close()
+ if err != nil {
+ runner.CrunchLog.Printf("error closing stdout logs: %v", err)
+ }
- if runner.statReporter != nil {
- runner.statReporter.Stop()
- closeerr = runner.statLogger.Close()
- if closeerr != nil {
- runner.CrunchLog.Printf("While closing crunchstat logs: %v", closeerr)
- }
- }
+ err = runner.Stderr.Close()
+ if err != nil {
+ runner.CrunchLog.Printf("error closing stderr logs: %v", err)
+ }
- close(runner.loggingDone)
- return
+ if runner.statReporter != nil {
+ runner.statReporter.Stop()
+ err = runner.statLogger.Close()
+ if err != nil {
+ runner.CrunchLog.Printf("error closing crunchstat logs: %v", err)
}
}
}
-func (runner *ContainerRunner) StartCrunchstat() {
+func (runner *ContainerRunner) stopHoststat() error {
+ if runner.hoststatReporter == nil {
+ return nil
+ }
+ runner.hoststatReporter.Stop()
+ err := runner.hoststatLogger.Close()
+ if err != nil {
+ return fmt.Errorf("error closing hoststat logs: %v", err)
+ }
+ return nil
+}
+
+func (runner *ContainerRunner) startHoststat() {
+ runner.hoststatLogger = NewThrottledLogger(runner.NewLogWriter("hoststat"))
+ runner.hoststatReporter = &crunchstat.Reporter{
+ Logger: log.New(runner.hoststatLogger, "", 0),
+ CgroupRoot: runner.cgroupRoot,
+ PollPeriod: runner.statInterval,
+ }
+ runner.hoststatReporter.Start()
+}
+
+func (runner *ContainerRunner) startCrunchstat() {
runner.statLogger = NewThrottledLogger(runner.NewLogWriter("crunchstat"))
runner.statReporter = &crunchstat.Reporter{
CID: runner.ContainerID,
}
checkErr(runner.CaptureOutput())
+ checkErr(runner.stopHoststat())
checkErr(runner.CommitLogs())
checkErr(runner.UpdateContainerFinal())
}()
if err != nil {
return
}
-
- // setup signal handling
runner.setupSignals()
+ runner.startHoststat()
// check for and/or load image
err = runner.LoadImage()
}
runner.finalState = "Cancelled"
- runner.StartCrunchstat()
+ runner.startCrunchstat()
err = runner.StartContainer()
if err != nil {
}()
errChan := make(chan error)
go func() {
- errChan <- v.bsClient.CreateBlockBlobFromReader(v.ContainerName, loc, uint64(len(block)), bufr, nil)
+ var body io.Reader = bufr
+ if len(block) == 0 {
+ // We must send a "Content-Length: 0" header,
+ // but the http client interprets
+ // ContentLength==0 as "unknown" unless it can
+ // confirm by introspection that Body will
+ // read 0 bytes.
+ body = http.NoBody
+ bufr.Close()
+ }
+ errChan <- v.bsClient.CreateBlockBlobFromReader(v.ContainerName, loc, uint64(len(block)), body, nil)
}()
select {
case <-ctx.Done():
func (c *azureBlobClient) CreateBlockBlobFromReader(cname, bname string, size uint64, rdr io.Reader, hdrs map[string]string) error {
c.stats.Tick(&c.stats.Ops, &c.stats.CreateOps)
- rdr = NewCountingReader(rdr, c.stats.TickOutBytes)
+ if size != 0 {
+ rdr = NewCountingReader(rdr, c.stats.TickOutBytes)
+ }
err := c.client.CreateBlockBlobFromReader(cname, bname, size, rdr, hdrs)
c.stats.TickErr(err)
return err
return
}
+ if (r.Method == "PUT" || r.Method == "POST") && r.Header.Get("Content-Length") == "" {
+ rw.WriteHeader(http.StatusLengthRequired)
+ return
+ }
+
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
userIDToUUID[uID] = u.UUID
if cfg.Verbose {
- log.Printf("Seen user %q (%s)", u.Username, u.Email)
+ log.Printf("Seen user %q (%s)", u.Username, u.UUID)
}
}
return err
}
log.Printf("Found %d remote groups", len(remoteGroups))
+ if cfg.Verbose {
+ for groupUUID := range remoteGroups {
+ log.Printf("- Group %q: %d users", remoteGroups[groupUUID].Group.Name, len(remoteGroups[groupUUID].PreviousMembers))
+ }
+ }
membershipsRemoved := 0
Operator: "=",
Operand: group.UUID,
}, {
- Attr: "head_kind",
- Operator: "=",
- Operand: "arvados#user",
+ Attr: "head_uuid",
+ Operator: "like",
+ Operand: "%-tpzed-%",
}},
}
// User -> Group filter
Operator: "=",
Operand: group.UUID,
}, {
- Attr: "tail_kind",
- Operator: "=",
- Operand: "arvados#user",
+ Attr: "tail_uuid",
+ Operator: "like",
+ Operand: "%-tpzed-%",
}},
}
g2uLinks, err := GetAll(cfg.Client, "links", g2uFilter, &LinkList{})
// RemoveMemberFromGroup remove all links related to the membership
func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
if cfg.Verbose {
- log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Email, user.UUID, group.Name, group.UUID)
+ log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Username, user.UUID, group.Name, group.UUID)
}
var links []interface{}
// Search for all group<->user links (both ways)
c.Assert(len(s.users), Not(Equals), 0)
}
-// Clean any membership link and remote group created by the test
func (s *TestSuite) TearDownTest(c *C) {
var dst interface{}
// Reset database to fixture state after every test run.
var _ = Suite(&TestSuite{})
-// MakeTempCVSFile creates a temp file with data as comma separated values
+// MakeTempCSVFile creates a temp file with data as comma separated values
func MakeTempCSVFile(data [][]string) (f *os.File, err error) {
f, err = ioutil.TempFile("", "test_sync_remote_groups")
if err != nil {
// The absence of a user membership on the CSV file implies its removal
func (s *TestSuite) TestMembershipRemoval(c *C) {
- activeUserEmail := s.users[arvadostest.ActiveUserUUID].Email
- activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ localUserEmail := s.users[arvadostest.ActiveUserUUID].Email
+ localUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ remoteUserEmail := s.users[arvadostest.FederatedActiveUserUUID].Email
+ remoteUserUUID := s.users[arvadostest.FederatedActiveUserUUID].UUID
data := [][]string{
- {"TestGroup1", activeUserEmail},
- {"TestGroup2", activeUserEmail},
+ {"TestGroup1", localUserEmail},
+ {"TestGroup1", remoteUserEmail},
+ {"TestGroup2", localUserEmail},
+ {"TestGroup2", remoteUserEmail},
}
tmpfile, err := MakeTempCSVFile(data)
c.Assert(err, IsNil)
groupUUID, err := RemoteGroupExists(s.cfg, groupName)
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, true)
}
- // New CSV with one previous membership missing
+ // New CSV with some previous membership missing
data = [][]string{
- {"TestGroup1", activeUserEmail},
+ {"TestGroup1", localUserEmail},
+ {"TestGroup2", remoteUserEmail},
}
tmpfile2, err := MakeTempCSVFile(data)
c.Assert(err, IsNil)
s.cfg.Path = tmpfile2.Name()
err = doMain(s.cfg)
c.Assert(err, IsNil)
- // Confirm TestGroup1 membership still exist
+ // Confirm TestGroup1 memberships
groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup1")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
- // Confirm TestGroup2 membership was removed
+ c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, true)
+ c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, false)
+ // Confirm TestGroup1 memberships
groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup2")
c.Assert(err, IsNil)
c.Assert(groupUUID, Not(Equals), "")
- c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, false)
+ c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, true)
}
// If a group doesn't exist on the system, create it before adding users
projects / arvados.git / commitdiff