X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0cbcf8cb892c6011ae162e840e1892602d8af940..bb99d90e25a47dff9405fb6f68be0779dd2d8989:/tools/sync-groups/sync-groups.go diff --git a/tools/sync-groups/sync-groups.go b/tools/sync-groups/sync-groups.go index 6b4781c354..e1e054a663 100644 --- a/tools/sync-groups/sync-groups.go +++ b/tools/sync-groups/sync-groups.go @@ -16,7 +16,8 @@ import ( "os" "strings" - "git.curoverse.com/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/lib/cmd" + "git.arvados.org/arvados.git/sdk/go/arvados" ) var version = "dev" @@ -26,11 +27,14 @@ type resourceList interface { GetItems() []interface{} } -// GroupInfo tracks previous and current members of a particular Group +// GroupPermissions maps permission levels on groups (can_read, can_write, can_manage) +type GroupPermissions map[string]bool + +// GroupInfo tracks previous and current member's permissions on a particular Group type GroupInfo struct { Group arvados.Group - PreviousMembers map[string]bool - CurrentMembers map[string]bool + PreviousMembers map[string]GroupPermissions + CurrentMembers map[string]GroupPermissions } // GetUserID returns the correct user id value depending on the selector @@ -116,6 +120,7 @@ type ConfigParams struct { Path string UserID string Verbose bool + CaseInsensitive bool ParentGroupUUID string ParentGroupName string SysUserUUID string @@ -130,16 +135,17 @@ func ParseFlags(config *ConfigParams) error { "username": true, } - flags := flag.NewFlagSet("arv-sync-groups", flag.ExitOnError) + flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError) // Set up usage message flags.Usage = func() { - usageStr := `Synchronize remote groups into Arvados from a CSV format file with 2 columns: - * 1st column: Group name - * 2nd column: User identifier` - fmt.Fprintf(os.Stderr, "%s\n\n", usageStr) - fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] \n\n", os.Args[0]) - fmt.Fprintf(os.Stderr, "Options:\n") + usageStr := `Synchronize remote groups into Arvados from a CSV format file with 3 columns: + * 1st: Group name + * 2nd: User identifier + * 3rd (Optional): User permission on the group: can_read, can_write or can_manage. (Default: can_write)` + fmt.Fprintf(flags.Output(), "%s\n\n", usageStr) + fmt.Fprintf(flags.Output(), "Usage:\n%s [OPTIONS] \n\n", os.Args[0]) + fmt.Fprintf(flags.Output(), "Options:\n") flags.PrintDefaults() } @@ -148,6 +154,10 @@ func ParseFlags(config *ConfigParams) error { "user-id", "email", "Attribute by which every user is identified. Valid values are: email and username.") + caseInsensitive := flags.Bool( + "case-insensitive", + false, + "Performs case insensitive matching on user IDs. Off by default.") verbose := flags.Bool( "verbose", false, @@ -161,12 +171,10 @@ func ParseFlags(config *ConfigParams) error { "", "Use given group UUID as a parent for the remote groups. Should be owned by the system user. If not specified, a group named '"+config.ParentGroupName+"' will be used (and created if nonexistant).") - // Parse args; omit the first arg which is the command name - flags.Parse(os.Args[1:]) - - // Print version information if requested - if *getVersion { - fmt.Printf("arv-sync-groups %s\n", version) + if ok, code := cmd.ParseFlags(flags, os.Args[0], os.Args[1:], "input-file.csv", os.Stderr); !ok { + os.Exit(code) + } else if *getVersion { + fmt.Printf("%s %s\n", os.Args[0], version) os.Exit(0) } @@ -192,6 +200,7 @@ func ParseFlags(config *ConfigParams) error { config.ParentGroupUUID = *parentGroupUUID config.UserID = *userID config.Verbose = *verbose + config.CaseInsensitive = *caseInsensitive return nil } @@ -217,13 +226,14 @@ func SetParentGroup(cfg *ConfigParams) error { return fmt.Errorf("error searching for parent group: %s", err) } if len(gl.Items) == 0 { - // Default parent group not existant, create one. + // Default parent group does not exist, create it. if cfg.Verbose { log.Println("Default parent group not found, creating...") } groupData := map[string]string{ - "name": cfg.ParentGroupName, - "owner_uuid": cfg.SysUserUUID, + "name": cfg.ParentGroupName, + "owner_uuid": cfg.SysUserUUID, + "group_class": "role", } if err := CreateGroup(cfg, &parentGroup, groupData); err != nil { return fmt.Errorf("error creating system user owned group named %q: %s", groupData["name"], err) @@ -270,7 +280,13 @@ func GetConfig() (config ConfigParams, err error) { if !u.IsActive || !u.IsAdmin { return config, fmt.Errorf("current user (%s) is not an active admin user", u.UUID) } - config.SysUserUUID = u.UUID[:12] + "000000000000000" + + var ac struct{ ClusterID string } + err = config.Client.RequestAndDecode(&ac, "GET", "arvados/v1/config", nil, nil) + if err != nil { + return config, fmt.Errorf("error getting the exported config: %s", err) + } + config.SysUserUUID = ac.ClusterID + "-tpzed-000000000000000" // Set up remote groups' parent if err = SetParentGroup(&config); err != nil { @@ -288,7 +304,11 @@ func doMain(cfg *ConfigParams) error { } defer f.Close() - log.Printf("arv-sync-groups %s started. Using %q as users id and parent group UUID %q", version, cfg.UserID, cfg.ParentGroupUUID) + iCaseLog := "" + if cfg.UserID == "username" && cfg.CaseInsensitive { + iCaseLog = " - username matching requested to be case-insensitive" + } + log.Printf("%s %s started. Using %q as users id and parent group UUID %q%s", os.Args[0], version, cfg.UserID, cfg.ParentGroupUUID, iCaseLog) // Get the complete user list to minimize API Server requests allUsers := make(map[string]arvados.User) @@ -305,9 +325,15 @@ func doMain(cfg *ConfigParams) error { if err != nil { return err } + if cfg.UserID == "username" && uID != "" && cfg.CaseInsensitive { + uID = strings.ToLower(uID) + if uuid, found := userIDToUUID[uID]; found { + return fmt.Errorf("case insensitive collision for username %q between %q and %q", uID, u.UUID, uuid) + } + } userIDToUUID[uID] = u.UUID if cfg.Verbose { - log.Printf("Seen user %q (%s)", u.Username, u.Email) + log.Printf("Seen user %q (%s)", u.Username, u.UUID) } } @@ -317,6 +343,11 @@ func doMain(cfg *ConfigParams) error { return err } log.Printf("Found %d remote groups", len(remoteGroups)) + if cfg.Verbose { + for groupUUID := range remoteGroups { + log.Printf("- Group %q: %d users", remoteGroups[groupUUID].Group.Name, len(remoteGroups[groupUUID].PreviousMembers)) + } + } membershipsRemoved := 0 @@ -329,16 +360,30 @@ func doMain(cfg *ConfigParams) error { // Remove previous members not listed on this run for groupUUID := range remoteGroups { gi := remoteGroups[groupUUID] - evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers) + evictedMemberPerms := subtract(gi.PreviousMembers, gi.CurrentMembers) groupName := gi.Group.Name - if len(evictedMembers) > 0 { - log.Printf("Removing %d users from group %q", len(evictedMembers), groupName) + if len(evictedMemberPerms) > 0 { + log.Printf("Removing permissions from %d users on group %q", len(evictedMemberPerms), groupName) } - for evictedUser := range evictedMembers { - if err := RemoveMemberFromGroup(cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil { + for member := range evictedMemberPerms { + var perms []string + completeMembershipRemoval := false + if _, ok := gi.CurrentMembers[member]; !ok { + completeMembershipRemoval = true + membershipsRemoved++ + } else { + // Collect which user->group permission links should be removed + for p := range evictedMemberPerms[member] { + if evictedMemberPerms[member][p] { + perms = append(perms, p) + } + } + membershipsRemoved += len(perms) + } + if err := RemoveMemberLinksFromGroup(cfg, allUsers[userIDToUUID[member]], + perms, completeMembershipRemoval, gi.Group); err != nil { return err } - membershipsRemoved++ } } log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped) @@ -357,7 +402,8 @@ func ProcessFile( ) (groupsCreated, membersAdded, membersSkipped int, err error) { lineNo := 0 csvReader := csv.NewReader(f) - csvReader.FieldsPerRecord = 2 + // Allow variable number of fields. + csvReader.FieldsPerRecord = -1 for { record, e := csvReader.Read() if e == io.EOF { @@ -368,10 +414,27 @@ func ProcessFile( err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo) return } + // Only allow 2 or 3 fields per record for backwards compatibility. + if len(record) < 2 || len(record) > 3 { + err = fmt.Errorf("error parsing %q, line %d: found %d fields but only 2 or 3 are allowed", cfg.Path, lineNo, len(record)) + return + } groupName := strings.TrimSpace(record[0]) groupMember := strings.TrimSpace(record[1]) // User ID (username or email) - if groupName == "" || groupMember == "" { - log.Printf("Warning: CSV record has at least one empty field (%s, %s). Skipping", groupName, groupMember) + groupPermission := "can_write" + if len(record) == 3 { + groupPermission = strings.ToLower(record[2]) + } + if groupName == "" || groupMember == "" || groupPermission == "" { + log.Printf("Warning: CSV record has at least one empty field (%s, %s, %s). Skipping", groupName, groupMember, groupPermission) + membersSkipped++ + continue + } + if cfg.UserID == "username" && cfg.CaseInsensitive { + groupMember = strings.ToLower(groupMember) + } + if !(groupPermission == "can_read" || groupPermission == "can_write" || groupPermission == "can_manage") { + log.Printf("Warning: 3rd field should be 'can_read', 'can_write' or 'can_manage'. Found: %q at line %d, skipping.", groupPermission, lineNo) membersSkipped++ continue } @@ -393,33 +456,43 @@ func ProcessFile( "group_class": "role", } if e := CreateGroup(cfg, &newGroup, groupData); e != nil { - err = fmt.Errorf("error creating group named %q: %s", groupName, err) + err = fmt.Errorf("error creating group named %q: %s", groupName, e) return } // Update cached group data groupNameToUUID[groupName] = newGroup.UUID remoteGroups[newGroup.UUID] = &GroupInfo{ Group: newGroup, - PreviousMembers: make(map[string]bool), // Empty set - CurrentMembers: make(map[string]bool), // Empty set + PreviousMembers: make(map[string]GroupPermissions), + CurrentMembers: make(map[string]GroupPermissions), } groupsCreated++ } // Both group & user exist, check if user is a member groupUUID := groupNameToUUID[groupName] gi := remoteGroups[groupUUID] - if !gi.PreviousMembers[groupMember] && !gi.CurrentMembers[groupMember] { + if !gi.PreviousMembers[groupMember][groupPermission] && !gi.CurrentMembers[groupMember][groupPermission] { if cfg.Verbose { log.Printf("Adding %q to group %q", groupMember, groupName) } - // User wasn't a member, but should be. - if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group); e != nil { + // User permissionwasn't there, but should be. Avoid duplicating the + // group->user link when necessary. + createG2ULink := true + if _, ok := gi.PreviousMembers[groupMember]; ok { + createG2ULink = false // User is already member of the group + } + if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group, groupPermission, createG2ULink); e != nil { err = e return } membersAdded++ } - gi.CurrentMembers[groupMember] = true + if _, ok := gi.CurrentMembers[groupMember]; ok { + gi.CurrentMembers[groupMember][groupPermission] = true + } else { + gi.CurrentMembers[groupMember] = GroupPermissions{groupPermission: true} + } + } return } @@ -439,19 +512,23 @@ func GetAll(c *arvados.Client, res string, params arvados.ResourceListParams, pa if page.Len() == 0 { break } - for _, i := range page.GetItems() { - allItems = append(allItems, i) - } + allItems = append(allItems, page.GetItems()...) params.Offset += page.Len() } return allItems, nil } -func subtract(setA map[string]bool, setB map[string]bool) map[string]bool { - result := make(map[string]bool) +func subtract(setA map[string]GroupPermissions, setB map[string]GroupPermissions) map[string]GroupPermissions { + result := make(map[string]GroupPermissions) for element := range setA { - if !setB[element] { - result[element] = true + if _, ok := setB[element]; !ok { + result[element] = setA[element] + } else { + for perm := range setA[element] { + if _, ok := setB[element][perm]; !ok { + result[element] = GroupPermissions{perm: true} + } + } } } return result @@ -474,17 +551,21 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot params := arvados.ResourceListParams{ Filters: []arvados.Filter{{ - Attr: "owner_uuid", + Attr: "tail_uuid", Operator: "=", Operand: cfg.ParentGroupUUID, }}, } - results, err := GetAll(cfg.Client, "groups", params, &GroupList{}) + results, err := GetAll(cfg.Client, "links", params, &LinkList{}) if err != nil { return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote groups: %s", err) } for _, item := range results { - group := item.(arvados.Group) + var group arvados.Group + err = GetGroup(cfg, &group, item.(arvados.Link).HeadUUID) + if err != nil { + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote group: %s", err) + } // Group -> User filter g2uFilter := arvados.ResourceListParams{ Filters: []arvados.Filter{{ @@ -504,8 +585,8 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot Operator: "=", Operand: group.UUID, }, { - Attr: "head_kind", - Operator: "=", + Attr: "head_uuid", + Operator: "is_a", Operand: "arvados#user", }}, } @@ -521,32 +602,37 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot Operand: "permission", }, { Attr: "name", - Operator: "=", - Operand: "can_write", + Operator: "in", + Operand: []string{"can_read", "can_write", "can_manage"}, }, { Attr: "head_uuid", Operator: "=", Operand: group.UUID, }, { - Attr: "tail_kind", - Operator: "=", + Attr: "tail_uuid", + Operator: "is_a", Operand: "arvados#user", }}, } g2uLinks, err := GetAll(cfg.Client, "links", g2uFilter, &LinkList{}) if err != nil { - return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_read) links for group %q: %s", group.Name, err) + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting group->user 'can_read' links for group %q: %s", group.Name, err) } u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{}) if err != nil { - return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_write) links for group %q: %s", group.Name, err) + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting user->group links for group %q: %s", group.Name, err) } - // Build a list of user ids (email or username) belonging to this group - membersSet := make(map[string]bool) - u2gLinkSet := make(map[string]bool) + // Build a list of user ids (email or username) belonging to this group. + membersSet := make(map[string]GroupPermissions) + u2gLinkSet := make(map[string]GroupPermissions) for _, l := range u2gLinks { - linkedMemberUUID := l.(arvados.Link).TailUUID - u2gLinkSet[linkedMemberUUID] = true + link := l.(arvados.Link) + // Also save the member's group access level. + if _, ok := u2gLinkSet[link.TailUUID]; ok { + u2gLinkSet[link.TailUUID][link.Name] = true + } else { + u2gLinkSet[link.TailUUID] = GroupPermissions{link.Name: true} + } } for _, item := range g2uLinks { link := item.(arvados.Link) @@ -564,63 +650,90 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot if err != nil { return remoteGroups, groupNameToUUID, err } - membersSet[memberID] = true + if cfg.UserID == "username" && cfg.CaseInsensitive { + memberID = strings.ToLower(memberID) + } + membersSet[memberID] = u2gLinkSet[link.HeadUUID] } remoteGroups[group.UUID] = &GroupInfo{ Group: group, PreviousMembers: membersSet, - CurrentMembers: make(map[string]bool), // Empty set + CurrentMembers: make(map[string]GroupPermissions), } groupNameToUUID[group.Name] = group.UUID } return remoteGroups, groupNameToUUID, nil } -// RemoveMemberFromGroup remove all links related to the membership -func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error { +// RemoveMemberLinksFromGroup remove all links related to the membership +func RemoveMemberLinksFromGroup(cfg *ConfigParams, user arvados.User, linkNames []string, completeRemoval bool, group arvados.Group) error { if cfg.Verbose { - log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Email, user.UUID, group.Name, group.UUID) + log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Username, user.UUID, group.Name, group.UUID) } var links []interface{} - // Search for all group<->user links (both ways) - for _, filterset := range [][]arvados.Filter{ - // Group -> User - {{ - Attr: "link_class", - Operator: "=", - Operand: "permission", - }, { - Attr: "tail_uuid", - Operator: "=", - Operand: group.UUID, - }, { - Attr: "head_uuid", - Operator: "=", - Operand: user.UUID, - }}, - // Group <- User - {{ - Attr: "link_class", - Operator: "=", - Operand: "permission", - }, { - Attr: "tail_uuid", - Operator: "=", - Operand: user.UUID, - }, { - Attr: "head_uuid", - Operator: "=", - Operand: group.UUID, - }}, - } { + var filters [][]arvados.Filter + if completeRemoval { + // Search for all group<->user links (both ways) + filters = [][]arvados.Filter{ + // Group -> User + {{ + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "tail_uuid", + Operator: "=", + Operand: group.UUID, + }, { + Attr: "head_uuid", + Operator: "=", + Operand: user.UUID, + }}, + // Group <- User + {{ + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "tail_uuid", + Operator: "=", + Operand: user.UUID, + }, { + Attr: "head_uuid", + Operator: "=", + Operand: group.UUID, + }}, + } + } else { + // Search only for the requested Group <- User permission links + filters = [][]arvados.Filter{ + {{ + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "tail_uuid", + Operator: "=", + Operand: user.UUID, + }, { + Attr: "head_uuid", + Operator: "=", + Operand: group.UUID, + }, { + Attr: "name", + Operator: "in", + Operand: linkNames, + }}, + } + } + + for _, filterset := range filters { l, err := GetAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &LinkList{}) if err != nil { userID, _ := GetUserID(user, cfg.UserID) return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", userID, group.Name, err) } - for _, link := range l { - links = append(links, link) - } + links = append(links, l...) } for _, item := range links { link := item.(arvados.Link) @@ -636,29 +749,32 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G } // AddMemberToGroup create membership links -func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error { +func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group, perm string, createG2ULink bool) error { var newLink arvados.Link - linkData := map[string]string{ - "owner_uuid": cfg.SysUserUUID, - "link_class": "permission", - "name": "can_read", - "tail_uuid": group.UUID, - "head_uuid": user.UUID, - } - if err := CreateLink(cfg, &newLink, linkData); err != nil { - userID, _ := GetUserID(user, cfg.UserID) - return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err) + var linkData map[string]string + if createG2ULink { + linkData = map[string]string{ + "owner_uuid": cfg.SysUserUUID, + "link_class": "permission", + "name": "can_read", + "tail_uuid": group.UUID, + "head_uuid": user.UUID, + } + if err := CreateLink(cfg, &newLink, linkData); err != nil { + userID, _ := GetUserID(user, cfg.UserID) + return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err) + } } linkData = map[string]string{ "owner_uuid": cfg.SysUserUUID, "link_class": "permission", - "name": "can_write", + "name": perm, "tail_uuid": user.UUID, "head_uuid": group.UUID, } if err := CreateLink(cfg, &newLink, linkData); err != nil { userID, _ := GetUserID(user, cfg.UserID) - return fmt.Errorf("error adding user %q -> group %q write permission: %s", userID, group.Name, err) + return fmt.Errorf("error adding user %q -> group %q %s permission: %s", userID, group.Name, perm, err) } return nil }