X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/26b510d785df7f548f41a11445c07df34b60328c..289d2cf581b59632369087388f6163f3979c5e86:/tools/arv-sync-groups/arv-sync-groups.go diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go index 87f89b6a45..d7efdefb6f 100644 --- a/tools/arv-sync-groups/arv-sync-groups.go +++ b/tools/arv-sync-groups/arv-sync-groups.go @@ -19,8 +19,6 @@ import ( "git.curoverse.com/arvados.git/sdk/go/arvados" ) -// const remoteGroupParentName string = "Externally synchronized groups" - type resourceList interface { Len() int GetItems() []interface{} @@ -81,21 +79,9 @@ func (l GroupList) GetItems() (out []interface{}) { return } -// Link is an arvados#link record -type Link struct { - UUID string `json:"uuid,omiempty"` - OwnerUUID string `json:"owner_uuid,omitempty"` - Name string `json:"name,omitempty"` - LinkClass string `json:"link_class,omitempty"` - HeadUUID string `json:"head_uuid,omitempty"` - HeadKind string `json:"head_kind,omitempty"` - TailUUID string `json:"tail_uuid,omitempty"` - TailKind string `json:"tail_kind,omitempty"` -} - // LinkList implements resourceList interface type LinkList struct { - Items []Link `json:"items"` + arvados.LinkList } // Len returns the amount of items this list holds @@ -112,7 +98,13 @@ func (l LinkList) GetItems() (out []interface{}) { } func main() { - if err := doMain(); err != nil { + // Parse & validate arguments, set up arvados client. + cfg, err := GetConfig() + if err != nil { + log.Fatalf("%v", err) + } + + if err := doMain(&cfg); err != nil { log.Fatalf("%v", err) } } @@ -135,11 +127,21 @@ func ParseFlags(config *ConfigParams) error { "email": true, // default "username": true, } + flags := flag.NewFlagSet("arv-sync-groups", flag.ExitOnError) - srcPath := flags.String( - "path", - "", - "Local file path containing a CSV format: GroupName,UserID") + + // Set up usage message + flags.Usage = func() { + usageStr := `Synchronize remote groups into Arvados from a CSV format file with 2 columns: + * 1st column: Group name + * 2nd column: User identifier` + fmt.Fprintf(os.Stderr, "%s\n\n", usageStr) + fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] \n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Options:\n") + flags.PrintDefaults() + } + + // Set up option flags userID := flags.String( "user-id", "email", @@ -156,9 +158,15 @@ func ParseFlags(config *ConfigParams) error { // Parse args; omit the first arg which is the command name flags.Parse(os.Args[1:]) + // Input file as a required positional argument + if flags.NArg() == 0 { + return fmt.Errorf("please provide a path to an input file") + } + srcPath := &os.Args[flags.NFlag()+1] + // Validations if *srcPath == "" { - return fmt.Errorf("please provide a path to an input file") + return fmt.Errorf("input file path invalid") } if !userIDOpts[*userID] { var options []string @@ -260,13 +268,7 @@ func GetConfig() (config ConfigParams, err error) { return config, nil } -func doMain() error { - // Parse & validate arguments, set up arvados client. - cfg, err := GetConfig() - if err != nil { - return err - } - +func doMain(cfg *ConfigParams) error { // Try opening the input file early, just in case there's a problem. f, err := os.Open(cfg.Path) if err != nil { @@ -298,38 +300,73 @@ func doMain() error { } // Get remote groups and their members - remoteGroups, groupNameToUUID, err := GetRemoteGroups(&cfg, allUsers) + remoteGroups, groupNameToUUID, err := GetRemoteGroups(cfg, allUsers) if err != nil { return err } log.Printf("Found %d remote groups", len(remoteGroups)) - groupsCreated := 0 - membershipsAdded := 0 membershipsRemoved := 0 - membershipsSkipped := 0 // Read the CSV file + groupsCreated, membershipsAdded, membershipsSkipped, err := ProcessFile(cfg, f, userIDToUUID, groupNameToUUID, remoteGroups, allUsers) + if err != nil { + return err + } + + // Remove previous members not listed on this run + for groupUUID := range remoteGroups { + gi := remoteGroups[groupUUID] + evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers) + groupName := gi.Group.Name + if len(evictedMembers) > 0 { + log.Printf("Removing %d users from group %q", len(evictedMembers), groupName) + } + for evictedUser := range evictedMembers { + if err := RemoveMemberFromGroup(cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil { + return err + } + membershipsRemoved++ + } + } + log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped) + + return nil +} + +// ProcessFile reads the CSV file and process every record +func ProcessFile( + cfg *ConfigParams, + f *os.File, + userIDToUUID map[string]string, + groupNameToUUID map[string]string, + remoteGroups map[string]*GroupInfo, + allUsers map[string]arvados.User, +) (groupsCreated, membersAdded, membersSkipped int, err error) { + lineNo := 0 csvReader := csv.NewReader(f) + csvReader.FieldsPerRecord = 2 for { - record, err := csvReader.Read() - if err == io.EOF { + record, e := csvReader.Read() + if e == io.EOF { break } - if err != nil { - return fmt.Errorf("error reading %q: %s", cfg.Path, err) + lineNo++ + if e != nil { + err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo) + return } groupName := strings.TrimSpace(record[0]) groupMember := strings.TrimSpace(record[1]) // User ID (username or email) if groupName == "" || groupMember == "" { log.Printf("Warning: CSV record has at least one empty field (%s, %s). Skipping", groupName, groupMember) - membershipsSkipped++ + membersSkipped++ continue } if _, found := userIDToUUID[groupMember]; !found { // User not present on the system, skip. log.Printf("Warning: there's no user with %s %q on the system, skipping.", cfg.UserID, groupMember) - membershipsSkipped++ + membersSkipped++ continue } if _, found := groupNameToUUID[groupName]; !found { @@ -339,11 +376,13 @@ func doMain() error { } var newGroup arvados.Group groupData := map[string]string{ - "name": groupName, - "owner_uuid": cfg.ParentGroupUUID, + "name": groupName, + "owner_uuid": cfg.ParentGroupUUID, + "group_class": "role", } - if err := CreateGroup(&cfg, &newGroup, groupData); err != nil { - return fmt.Errorf("error creating group named %q: %s", groupName, err) + if e := CreateGroup(cfg, &newGroup, groupData); e != nil { + err = fmt.Errorf("error creating group named %q: %s", groupName, err) + return } // Update cached group data groupNameToUUID[groupName] = newGroup.UUID @@ -362,32 +401,15 @@ func doMain() error { log.Printf("Adding %q to group %q", groupMember, groupName) } // User wasn't a member, but should be. - if err := AddMemberToGroup(&cfg, allUsers[userIDToUUID[groupMember]], gi.Group); err != nil { - return nil + if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group); e != nil { + err = e + return } - membershipsAdded++ + membersAdded++ } gi.CurrentMembers[groupMember] = true } - - // Remove previous members not listed on this run - for groupUUID := range remoteGroups { - gi := remoteGroups[groupUUID] - evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers) - groupName := gi.Group.Name - if len(evictedMembers) > 0 { - log.Printf("Removing %d users from group %q", len(evictedMembers), groupName) - } - for evictedUser := range evictedMembers { - if err := RemoveMemberFromGroup(&cfg, allUsers[evictedUser], gi.Group); err != nil { - return err - } - membershipsRemoved++ - } - } - log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped) - - return nil + return } // GetAll : Adds all objects of type 'resource' to the 'allItems' list @@ -488,7 +510,7 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot }, { Attr: "name", Operator: "=", - Operand: "manage", + Operand: "can_write", }, { Attr: "head_uuid", Operator: "=", @@ -505,17 +527,17 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot } u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{}) if err != nil { - return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (manage) links for group %q: %s", group.Name, err) + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_write) links for group %q: %s", group.Name, err) } // Build a list of user ids (email or username) belonging to this group membersSet := make(map[string]bool) u2gLinkSet := make(map[string]bool) for _, l := range u2gLinks { - linkedMemberUUID := l.(Link).TailUUID + linkedMemberUUID := l.(arvados.Link).TailUUID u2gLinkSet[linkedMemberUUID] = true } for _, item := range g2uLinks { - link := item.(Link) + link := item.(arvados.Link) // We may have received an old link pointing to a removed account. if _, found := allUsers[link.HeadUUID]; !found { continue @@ -589,12 +611,12 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G } } for _, item := range links { - link := item.(Link) + link := item.(arvados.Link) + userID, _ := GetUserID(user, cfg.UserID) if cfg.Verbose { - log.Printf("Removing permission link for %q on group %q", user.Email, group.Name) + log.Printf("Removing %q permission link for %q on group %q", link.Name, userID, group.Name) } if err := DeleteLink(cfg, link.UUID); err != nil { - userID, _ := GetUserID(user, cfg.UserID) return fmt.Errorf("error removing user %q from group %q: %s", userID, group.Name, err) } } @@ -603,7 +625,7 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G // AddMemberToGroup create membership links func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error { - var newLink Link + var newLink arvados.Link linkData := map[string]string{ "owner_uuid": cfg.SysUserUUID, "link_class": "permission", @@ -613,18 +635,18 @@ func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) } if err := CreateLink(cfg, &newLink, linkData); err != nil { userID, _ := GetUserID(user, cfg.UserID) - return fmt.Errorf("error adding group %q -> user %q read permission: %s", userID, user.Email, err) + return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err) } linkData = map[string]string{ "owner_uuid": cfg.SysUserUUID, "link_class": "permission", - "name": "manage", + "name": "can_write", "tail_uuid": user.UUID, "head_uuid": group.UUID, } if err := CreateLink(cfg, &newLink, linkData); err != nil { userID, _ := GetUserID(user, cfg.UserID) - return fmt.Errorf("error adding user %q -> group %q manage permission: %s", userID, group.Name, err) + return fmt.Errorf("error adding user %q -> group %q write permission: %s", userID, group.Name, err) } return nil } @@ -640,13 +662,16 @@ func GetGroup(cfg *ConfigParams, dst *arvados.Group, groupUUID string) error { } // CreateLink creates a link with linkData parameters, assigns it to dst -func CreateLink(cfg *ConfigParams, dst *Link, linkData map[string]string) error { +func CreateLink(cfg *ConfigParams, dst *arvados.Link, linkData map[string]string) error { return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil) } // DeleteLink deletes a link by its UUID func DeleteLink(cfg *ConfigParams, linkUUID string) error { - return cfg.Client.RequestAndDecode(&Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil) + if linkUUID == "" { + return fmt.Errorf("cannot delete link with invalid UUID: %q", linkUUID) + } + return cfg.Client.RequestAndDecode(&arvados.Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil) } // GetResourceList fetches res list using params