From e2c27eaae38a904a4b05d800affdc7860ee24e79 Mon Sep 17 00:00:00 2001 From: Lucas Di Pentima Date: Tue, 24 Oct 2017 22:11:31 -0300 Subject: [PATCH] 12018: Move remote group retrieving and member removal code to separate functions. Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- tools/arv-sync-groups/arv-sync-groups.go | 340 ++++++++++++----------- 1 file changed, 180 insertions(+), 160 deletions(-) diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go index f02e405a42..9a7daadcb6 100644 --- a/tools/arv-sync-groups/arv-sync-groups.go +++ b/tools/arv-sync-groups/arv-sync-groups.go @@ -26,7 +26,8 @@ type resourceList interface { GetItems() []interface{} } -type groupInfo struct { +// GroupInfo tracks previous and current members of a particular Group +type GroupInfo struct { Group arvados.Group PreviousMembers map[string]bool CurrentMembers map[string]bool @@ -93,17 +94,17 @@ type Link struct { } // LinkList implements resourceList interface -type linkList struct { +type LinkList struct { Items []Link `json:"items"` } // Len returns the amount of items this list holds -func (l linkList) Len() int { +func (l LinkList) Len() int { return len(l.Items) } // GetItems returns the list of items -func (l linkList) GetItems() (out []interface{}) { +func (l LinkList) GetItems() (out []interface{}) { for _, item := range l.Items { out = append(out, item) } @@ -278,7 +279,7 @@ func doMain() error { // Get the complete user list to minimize API Server requests allUsers := make(map[string]arvados.User) userIDToUUID := make(map[string]string) // Index by email or username - results, err := ListAll(cfg.Client, "users", arvados.ResourceListParams{}, &UserList{}) + results, err := GetAll(cfg.Client, "users", arvados.ResourceListParams{}, &UserList{}) if err != nil { return fmt.Errorf("error getting user list: %s", err) } @@ -297,108 +298,9 @@ func doMain() error { } // Get remote groups and their members - remoteGroups := make(map[string]*groupInfo) - groupNameToUUID := make(map[string]string) // Index by group name - params := arvados.ResourceListParams{ - Filters: []arvados.Filter{{ - Attr: "owner_uuid", - Operator: "=", - Operand: cfg.ParentGroupUUID, - }}, - } - results, err = ListAll(cfg.Client, "groups", params, &GroupList{}) + remoteGroups, groupNameToUUID, err := GetRemoteGroups(&cfg, allUsers) if err != nil { - return fmt.Errorf("error getting remote groups: %s", err) - } - for _, item := range results { - group := item.(arvados.Group) - // Group -> User filter - g2uFilter := arvados.ResourceListParams{ - Filters: []arvados.Filter{{ - Attr: "owner_uuid", - Operator: "=", - Operand: cfg.SysUserUUID, - }, { - Attr: "link_class", - Operator: "=", - Operand: "permission", - }, { - Attr: "name", - Operator: "=", - Operand: "can_read", - }, { - Attr: "tail_uuid", - Operator: "=", - Operand: group.UUID, - }, { - Attr: "head_kind", - Operator: "=", - Operand: "arvados#user", - }}, - } - // User -> Group filter - u2gFilter := arvados.ResourceListParams{ - Filters: []arvados.Filter{{ - Attr: "owner_uuid", - Operator: "=", - Operand: cfg.SysUserUUID, - }, { - Attr: "link_class", - Operator: "=", - Operand: "permission", - }, { - Attr: "name", - Operator: "=", - Operand: "manage", - }, { - Attr: "head_uuid", - Operator: "=", - Operand: group.UUID, - }, { - Attr: "tail_kind", - Operator: "=", - Operand: "arvados#user", - }}, - } - g2uLinks, err := ListAll(cfg.Client, "links", g2uFilter, &linkList{}) - if err != nil { - return fmt.Errorf("error getting member (can_read) links for group %q: %s", group.Name, err) - } - u2gLinks, err := ListAll(cfg.Client, "links", u2gFilter, &linkList{}) - if err != nil { - return fmt.Errorf("error getting member (manage) links for group %q: %s", group.Name, err) - } - // Build a list of user ids (email or username) belonging to this group - membersSet := make(map[string]bool) - u2gLinkSet := make(map[string]bool) - for _, l := range u2gLinks { - linkedMemberUUID := l.(Link).TailUUID - u2gLinkSet[linkedMemberUUID] = true - } - for _, item := range g2uLinks { - link := item.(Link) - // We may have received an old link pointing to a removed account. - if _, found := allUsers[link.HeadUUID]; !found { - continue - } - // The matching User -> Group link may not exist if the link - // creation failed on a previous run. If that's the case, don't - // include this account on the previous members list. - if _, found := u2gLinkSet[link.HeadUUID]; !found { - continue - } - memberID, err := GetUserID(allUsers[link.HeadUUID], cfg.UserID) - if err != nil { - return err - } - membersSet[memberID] = true - } - remoteGroups[group.UUID] = &groupInfo{ - Group: group, - PreviousMembers: membersSet, - CurrentMembers: make(map[string]bool), // Empty set - } - groupNameToUUID[group.Name] = group.UUID + return err } log.Printf("Found %d remote groups", len(remoteGroups)) @@ -407,6 +309,7 @@ func doMain() error { membershipsRemoved := 0 membershipsSkipped := 0 + // Read the CSV file csvReader := csv.NewReader(f) for { record, err := csvReader.Read() @@ -444,7 +347,7 @@ func doMain() error { } // Update cached group data groupNameToUUID[groupName] = newGroup.UUID - remoteGroups[newGroup.UUID] = &groupInfo{ + remoteGroups[newGroup.UUID] = &GroupInfo{ Group: newGroup, PreviousMembers: make(map[string]bool), // Empty set CurrentMembers: make(map[string]bool), // Empty set @@ -494,57 +397,8 @@ func doMain() error { log.Printf("Removing %d users from group %q", len(evictedMembers), groupName) } for evictedUser := range evictedMembers { - if cfg.Verbose { - log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", evictedUser, userIDToUUID[evictedUser], groupName, groupUUID) - } - var links []interface{} - // Search for all group<->user links (both ways) - for _, filterset := range [][]arvados.Filter{ - // Group -> User - {{ - Attr: "link_class", - Operator: "=", - Operand: "permission", - }, { - Attr: "tail_uuid", - Operator: "=", - Operand: groupUUID, - }, { - Attr: "head_uuid", - Operator: "=", - Operand: userIDToUUID[evictedUser], - }}, - // Group <- User - {{ - Attr: "link_class", - Operator: "=", - Operand: "permission", - }, { - Attr: "tail_uuid", - Operator: "=", - Operand: userIDToUUID[evictedUser], - }, { - Attr: "head_uuid", - Operator: "=", - Operand: groupUUID, - }}, - } { - l, err := ListAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &linkList{}) - if err != nil { - return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", evictedUser, groupName, err) - } - for _, link := range l { - links = append(links, link) - } - } - for _, item := range links { - link := item.(Link) - if cfg.Verbose { - log.Printf("Removing permission link for %q on group %q", evictedUser, gi.Group.Name) - } - if err := cfg.Client.RequestAndDecode(&link, "DELETE", "/arvados/v1/links/"+link.UUID, nil, nil); err != nil { - return fmt.Errorf("error removing user %q from group %q: %s", evictedUser, groupName, err) - } + if err := RemoveMemberFromGroup(&cfg, allUsers[evictedUser], gi.Group); err != nil { + return err } membershipsRemoved++ } @@ -554,8 +408,8 @@ func doMain() error { return nil } -// ListAll : Adds all objects of type 'resource' to the 'allItems' list -func ListAll(c *arvados.Client, res string, params arvados.ResourceListParams, page resourceList) (allItems []interface{}, err error) { +// GetAll : Adds all objects of type 'resource' to the 'allItems' list +func GetAll(c *arvados.Client, res string, params arvados.ResourceListParams, page resourceList) (allItems []interface{}, err error) { // Use the maximum page size the server allows limit := 1<<31 - 1 params.Limit = &limit @@ -596,3 +450,169 @@ func jsonReader(rscName string, ob interface{}) io.Reader { v[rscName] = []string{string(j)} return bytes.NewBufferString(v.Encode()) } + +// GetRemoteGroups fetches all remote groups with their members +func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remoteGroups map[string]*GroupInfo, groupNameToUUID map[string]string, err error) { + remoteGroups = make(map[string]*GroupInfo) + groupNameToUUID = make(map[string]string) // Index by group name + + params := arvados.ResourceListParams{ + Filters: []arvados.Filter{{ + Attr: "owner_uuid", + Operator: "=", + Operand: cfg.ParentGroupUUID, + }}, + } + results, err := GetAll(cfg.Client, "groups", params, &GroupList{}) + if err != nil { + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote groups: %s", err) + } + for _, item := range results { + group := item.(arvados.Group) + // Group -> User filter + g2uFilter := arvados.ResourceListParams{ + Filters: []arvados.Filter{{ + Attr: "owner_uuid", + Operator: "=", + Operand: cfg.SysUserUUID, + }, { + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "name", + Operator: "=", + Operand: "can_read", + }, { + Attr: "tail_uuid", + Operator: "=", + Operand: group.UUID, + }, { + Attr: "head_kind", + Operator: "=", + Operand: "arvados#user", + }}, + } + // User -> Group filter + u2gFilter := arvados.ResourceListParams{ + Filters: []arvados.Filter{{ + Attr: "owner_uuid", + Operator: "=", + Operand: cfg.SysUserUUID, + }, { + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "name", + Operator: "=", + Operand: "manage", + }, { + Attr: "head_uuid", + Operator: "=", + Operand: group.UUID, + }, { + Attr: "tail_kind", + Operator: "=", + Operand: "arvados#user", + }}, + } + g2uLinks, err := GetAll(cfg.Client, "links", g2uFilter, &LinkList{}) + if err != nil { + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_read) links for group %q: %s", group.Name, err) + } + u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{}) + if err != nil { + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (manage) links for group %q: %s", group.Name, err) + } + // Build a list of user ids (email or username) belonging to this group + membersSet := make(map[string]bool) + u2gLinkSet := make(map[string]bool) + for _, l := range u2gLinks { + linkedMemberUUID := l.(Link).TailUUID + u2gLinkSet[linkedMemberUUID] = true + } + for _, item := range g2uLinks { + link := item.(Link) + // We may have received an old link pointing to a removed account. + if _, found := allUsers[link.HeadUUID]; !found { + continue + } + // The matching User -> Group link may not exist if the link + // creation failed on a previous run. If that's the case, don't + // include this account on the "previous members" list. + if _, found := u2gLinkSet[link.HeadUUID]; !found { + continue + } + memberID, err := GetUserID(allUsers[link.HeadUUID], cfg.UserID) + if err != nil { + return remoteGroups, groupNameToUUID, err + } + membersSet[memberID] = true + } + remoteGroups[group.UUID] = &GroupInfo{ + Group: group, + PreviousMembers: membersSet, + CurrentMembers: make(map[string]bool), // Empty set + } + groupNameToUUID[group.Name] = group.UUID + } + return remoteGroups, groupNameToUUID, nil +} + +// RemoveMemberFromGroup remove all links related to the membership +func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error { + if cfg.Verbose { + log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Email, user.UUID, group.Name, group.UUID) + } + var links []interface{} + // Search for all group<->user links (both ways) + for _, filterset := range [][]arvados.Filter{ + // Group -> User + {{ + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "tail_uuid", + Operator: "=", + Operand: group.UUID, + }, { + Attr: "head_uuid", + Operator: "=", + Operand: user.UUID, + }}, + // Group <- User + {{ + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "tail_uuid", + Operator: "=", + Operand: user.UUID, + }, { + Attr: "head_uuid", + Operator: "=", + Operand: group.UUID, + }}, + } { + l, err := GetAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &LinkList{}) + if err != nil { + return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", user.Email, group.Name, err) + } + for _, link := range l { + links = append(links, link) + } + } + for _, item := range links { + link := item.(Link) + if cfg.Verbose { + log.Printf("Removing permission link for %q on group %q", user.Email, group.Name) + } + if err := cfg.Client.RequestAndDecode(&link, "DELETE", "/arvados/v1/links/"+link.UUID, nil, nil); err != nil { + return fmt.Errorf("error removing user %q from group %q: %s", user.Email, group.Name, err) + } + } + return nil +} -- 2.39.5