X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/00a299d80ee03e3ae227eb8e237a43f29a6cc667..289d2cf581b59632369087388f6163f3979c5e86:/tools/arv-sync-groups/arv-sync-groups.go diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go index b8c94ed5df..d7efdefb6f 100644 --- a/tools/arv-sync-groups/arv-sync-groups.go +++ b/tools/arv-sync-groups/arv-sync-groups.go @@ -5,37 +5,34 @@ package main import ( + "bytes" "encoding/csv" + "encoding/json" "flag" "fmt" "io" "log" + "net/url" "os" "strings" "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" ) type resourceList interface { - items() []interface{} - itemsAvailable() int - offset() int + Len() int + GetItems() []interface{} } -type groupInfo struct { - Group group +// GroupInfo tracks previous and current members of a particular Group +type GroupInfo struct { + Group arvados.Group PreviousMembers map[string]bool CurrentMembers map[string]bool } -type user struct { - UUID string `json:"uuid,omitempty"` - Email string `json:"email,omitempty"` - Username string `json:"username,omitempty"` -} - -func (u user) GetID(idSelector string) (string, error) { +// GetUserID returns the correct user id value depending on the selector +func GetUserID(u arvados.User, idSelector string) (string, error) { switch idSelector { case "email": return u.Email, nil @@ -46,207 +43,178 @@ func (u user) GetID(idSelector string) (string, error) { } } -// userList implements resourceList interface -type userList struct { - Items []user `json:"items"` - ItemsAvailable int `json:"items_available"` - Offset int `json:"offset"` +// UserList implements resourceList interface +type UserList struct { + arvados.UserList +} + +// Len returns the amount of items this list holds +func (l UserList) Len() int { + return len(l.Items) } -func (l userList) items() []interface{} { - var out []interface{} +// GetItems returns the list of items +func (l UserList) GetItems() (out []interface{}) { for _, item := range l.Items { out = append(out, item) } - return out + return } -func (l userList) itemsAvailable() int { - return l.ItemsAvailable +// GroupList implements resourceList interface +type GroupList struct { + arvados.GroupList } -func (l userList) offset() int { - return l.Offset +// Len returns the amount of items this list holds +func (l GroupList) Len() int { + return len(l.Items) } -type group struct { - UUID string `json:"uuid,omitempty"` - Name string `json:"name,omitempty"` - OwnerUUID string `json:"owner_uuid,omitempty"` -} - -// groupList implements resourceList interface -type groupList struct { - Items []group `json:"items"` - ItemsAvailable int `json:"items_available"` - Offset int `json:"offset"` -} - -func (l groupList) items() []interface{} { - var out []interface{} +// GetItems returns the list of items +func (l GroupList) GetItems() (out []interface{}) { for _, item := range l.Items { out = append(out, item) } - return out -} - -func (l groupList) itemsAvailable() int { - return l.ItemsAvailable + return } -func (l groupList) offset() int { - return l.Offset +// LinkList implements resourceList interface +type LinkList struct { + arvados.LinkList } -type link struct { - UUID string `json:"uuid, omiempty"` - Name string `json:"name,omitempty"` - LinkClass string `json:"link_class,omitempty"` - HeadUUID string `json:"head_uuid,omitempty"` - HeadKind string `json:"head_kind,omitempty"` - TailUUID string `json:"tail_uuid,omitempty"` - TailKind string `json:"tail_kind,omitempty"` +// Len returns the amount of items this list holds +func (l LinkList) Len() int { + return len(l.Items) } -// linkList implements resourceList interface -type linkList struct { - Items []link `json:"items"` - ItemsAvailable int `json:"items_available"` - Offset int `json:"offset"` -} - -func (l linkList) items() []interface{} { - var out []interface{} +// GetItems returns the list of items +func (l LinkList) GetItems() (out []interface{}) { for _, item := range l.Items { out = append(out, item) } - return out -} - -func (l linkList) itemsAvailable() int { - return l.ItemsAvailable -} - -func (l linkList) offset() int { - return l.Offset + return } func main() { - err := doMain() + // Parse & validate arguments, set up arvados client. + cfg, err := GetConfig() if err != nil { log.Fatalf("%v", err) } + + if err := doMain(&cfg); err != nil { + log.Fatalf("%v", err) + } } -func doMain() error { - const groupTag string = "remote_group" - const remoteGroupParentName string = "Externally synchronized groups" - userIDOpts := []string{"email", "username"} +// ConfigParams holds configuration data for this tool +type ConfigParams struct { + Path string + UserID string + Verbose bool + ParentGroupUUID string + ParentGroupName string + SysUserUUID string + Client *arvados.Client +} + +// ParseFlags parses and validates command line arguments +func ParseFlags(config *ConfigParams) error { + // Acceptable attributes to identify a user on the CSV file + userIDOpts := map[string]bool{ + "email": true, // default + "username": true, + } flags := flag.NewFlagSet("arv-sync-groups", flag.ExitOnError) - srcPath := flags.String( - "path", - "", - "Local file path containing a CSV format.") + // Set up usage message + flags.Usage = func() { + usageStr := `Synchronize remote groups into Arvados from a CSV format file with 2 columns: + * 1st column: Group name + * 2nd column: User identifier` + fmt.Fprintf(os.Stderr, "%s\n\n", usageStr) + fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] \n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Options:\n") + flags.PrintDefaults() + } + // Set up option flags userID := flags.String( "user-id", "email", - "Attribute by which every user is identified. "+ - "Valid values are: email (the default) and username.") - + "Attribute by which every user is identified. Valid values are: email and username.") verbose := flags.Bool( "verbose", false, - "Log informational messages. By default is deactivated.") - - retries := flags.Int( - "retries", - 3, - "Maximum number of times to retry server requests that encounter "+ - "temporary failures (e.g., server down). Default 3.") - + "Log informational messages. Off by default.") parentGroupUUID := flags.String( "parent-group-uuid", "", - "Use given group UUID as a parent for the remote groups. Should "+ - "be owned by the system user. If not specified, a group named '"+ - remoteGroupParentName+"' will be used (and created if nonexistant).") + "Use given group UUID as a parent for the remote groups. Should be owned by the system user. If not specified, a group named '"+config.ParentGroupName+"' will be used (and created if nonexistant).") // Parse args; omit the first arg which is the command name flags.Parse(os.Args[1:]) - // Validations - if *retries < 0 { - return fmt.Errorf("retry quantity must be >= 0") - } - - if *srcPath == "" { + // Input file as a required positional argument + if flags.NArg() == 0 { return fmt.Errorf("please provide a path to an input file") } + srcPath := &os.Args[flags.NFlag()+1] - // Try opening the input file early, just in case there's problems. - f, err := os.Open(*srcPath) - if err != nil { - return fmt.Errorf("%s", err) + // Validations + if *srcPath == "" { + return fmt.Errorf("input file path invalid") } - defer f.Close() - - validUserID := false - for _, opt := range userIDOpts { - if *userID == opt { - validUserID = true + if !userIDOpts[*userID] { + var options []string + for opt := range userIDOpts { + options = append(options, opt) } - } - if !validUserID { - return fmt.Errorf("user ID must be one of: %s", - strings.Join(userIDOpts, ", ")) + return fmt.Errorf("user ID must be one of: %s", strings.Join(options, ", ")) } - // Arvados Client setup - ac := arvados.NewClientFromEnv() - arv, err := arvadosclient.New(ac) - if err != nil { - return fmt.Errorf("error setting up arvados client %s", err) - } - arv.Retries = *retries + config.Path = *srcPath + config.ParentGroupUUID = *parentGroupUUID + config.UserID = *userID + config.Verbose = *verbose - // Check current user permissions & get System user's UUID - u, err := ac.CurrentUser() - if err != nil { - return fmt.Errorf("error getting the current user: %s", err) - } - if !u.IsActive || !u.IsAdmin { - return fmt.Errorf("current user (%s) is not an active admin user", u.UUID) - } - sysUserUUID := u.UUID[:12] + "000000000000000" + return nil +} - // Find/create parent group - var parentGroup group - if *parentGroupUUID == "" { +// SetParentGroup finds/create parent group of all remote groups +func SetParentGroup(cfg *ConfigParams) error { + var parentGroup arvados.Group + if cfg.ParentGroupUUID == "" { // UUID not provided, search for preexisting parent group - var gl groupList - err := arv.List("groups", arvadosclient.Dict{ - "filters": [][]string{ - {"name", "=", remoteGroupParentName}, - {"owner_uuid", "=", sysUserUUID}}, - }, &gl) - if err != nil { + var gl GroupList + params := arvados.ResourceListParams{ + Filters: []arvados.Filter{{ + Attr: "name", + Operator: "=", + Operand: cfg.ParentGroupName, + }, { + Attr: "owner_uuid", + Operator: "=", + Operand: cfg.SysUserUUID, + }}, + } + if err := cfg.Client.RequestAndDecode(&gl, "GET", "/arvados/v1/groups", nil, params); err != nil { return fmt.Errorf("error searching for parent group: %s", err) } if len(gl.Items) == 0 { // Default parent group not existant, create one. - if *verbose { + if cfg.Verbose { log.Println("Default parent group not found, creating...") } - err := arv.Create("groups", arvadosclient.Dict{ - "group": arvadosclient.Dict{ - "name": remoteGroupParentName, - "owner_uuid": sysUserUUID}, - }, &parentGroup) - if err != nil { - return fmt.Errorf("error creating system user owned group named %q: %s", remoteGroupParentName, err) + groupData := map[string]string{ + "name": cfg.ParentGroupName, + "owner_uuid": cfg.SysUserUUID, + } + if err := CreateGroup(cfg, &parentGroup, groupData); err != nil { + return fmt.Errorf("error creating system user owned group named %q: %s", groupData["name"], err) } } else if len(gl.Items) == 1 { // Default parent group found. @@ -254,161 +222,172 @@ func doMain() error { } else { // This should never happen, as there's an unique index for // (owner_uuid, name) on groups. - return fmt.Errorf("found %d groups owned by system user and named %q", len(gl.Items), remoteGroupParentName) + return fmt.Errorf("bug: found %d groups owned by system user and named %q", len(gl.Items), cfg.ParentGroupName) } + cfg.ParentGroupUUID = parentGroup.UUID } else { // UUID provided. Check if exists and if it's owned by system user - err := arv.Get("groups", *parentGroupUUID, arvadosclient.Dict{}, &parentGroup) - if err != nil { - return fmt.Errorf("error searching for parent group with UUID %q: %s", *parentGroupUUID, err) - } - if parentGroup.UUID == "" { - return fmt.Errorf("parent group with UUID %q not found", *parentGroupUUID) + if err := GetGroup(cfg, &parentGroup, cfg.ParentGroupUUID); err != nil { + return fmt.Errorf("error searching for parent group with UUID %q: %s", cfg.ParentGroupUUID, err) } - if parentGroup.OwnerUUID != sysUserUUID { - return fmt.Errorf("parent group %q (%s) must be owned by system user", parentGroup.Name, *parentGroupUUID) + if parentGroup.OwnerUUID != cfg.SysUserUUID { + return fmt.Errorf("parent group %q (%s) must be owned by system user", parentGroup.Name, cfg.ParentGroupUUID) } } + return nil +} + +// GetConfig sets up a ConfigParams struct +func GetConfig() (config ConfigParams, err error) { + config.ParentGroupName = "Externally synchronized groups" + + // Command arguments + err = ParseFlags(&config) + if err != nil { + return config, err + } + + // Arvados Client setup + config.Client = arvados.NewClientFromEnv() + + // Check current user permissions & get System user's UUID + u, err := config.Client.CurrentUser() + if err != nil { + return config, fmt.Errorf("error getting the current user: %s", err) + } + if !u.IsActive || !u.IsAdmin { + return config, fmt.Errorf("current user (%s) is not an active admin user", u.UUID) + } + config.SysUserUUID = u.UUID[:12] + "000000000000000" + + // Set up remote groups' parent + if err = SetParentGroup(&config); err != nil { + return config, err + } + + return config, nil +} + +func doMain(cfg *ConfigParams) error { + // Try opening the input file early, just in case there's a problem. + f, err := os.Open(cfg.Path) + if err != nil { + return fmt.Errorf("%s", err) + } + defer f.Close() - log.Printf("Group sync starting. Using %q as users id and parent group UUID %q", *userID, parentGroup.UUID) + log.Printf("Group sync starting. Using %q as users id and parent group UUID %q", cfg.UserID, cfg.ParentGroupUUID) // Get the complete user list to minimize API Server requests - allUsers := make(map[string]user) + allUsers := make(map[string]arvados.User) userIDToUUID := make(map[string]string) // Index by email or username - results, err := ListAll(arv, "users", arvadosclient.Dict{}, &userList{}) + results, err := GetAll(cfg.Client, "users", arvados.ResourceListParams{}, &UserList{}) if err != nil { return fmt.Errorf("error getting user list: %s", err) } log.Printf("Found %d users", len(results)) for _, item := range results { - u := item.(user) + u := item.(arvados.User) allUsers[u.UUID] = u - uID, err := u.GetID(*userID) + uID, err := GetUserID(u, cfg.UserID) if err != nil { return err } userIDToUUID[uID] = u.UUID - if *verbose { + if cfg.Verbose { log.Printf("Seen user %q (%s)", u.Username, u.Email) } } - // Request all UUIDs for groups tagged as remote - remoteGroupUUIDs := make(map[string]bool) - results, err = ListAll(arv, "links", arvadosclient.Dict{ - "filters": [][]string{ - {"link_class", "=", "tag"}, - {"name", "=", groupTag}, - {"head_kind", "=", "arvados#group"}, - }, - }, &linkList{}) + // Get remote groups and their members + remoteGroups, groupNameToUUID, err := GetRemoteGroups(cfg, allUsers) if err != nil { - return fmt.Errorf("error getting remote group UUIDs: %s", err) - } - for _, item := range results { - link := item.(link) - remoteGroupUUIDs[link.HeadUUID] = true + return err } - // Get remote groups and their members - var uuidList []string - for uuid := range remoteGroupUUIDs { - uuidList = append(uuidList, uuid) - } - remoteGroups := make(map[string]*groupInfo) - groupNameToUUID := make(map[string]string) // Index by group name - results, err = ListAll(arv, "groups", arvadosclient.Dict{ - "filters": [][]interface{}{ - {"uuid", "in", uuidList}, - {"owner_uuid", "=", parentGroup.UUID}, - }, - }, &groupList{}) + log.Printf("Found %d remote groups", len(remoteGroups)) + + membershipsRemoved := 0 + + // Read the CSV file + groupsCreated, membershipsAdded, membershipsSkipped, err := ProcessFile(cfg, f, userIDToUUID, groupNameToUUID, remoteGroups, allUsers) if err != nil { - return fmt.Errorf("error getting remote groups by UUID: %s", err) + return err } - for _, item := range results { - group := item.(group) - results, err := ListAll(arv, "links", arvadosclient.Dict{ - "filters": [][]string{ - {"link_class", "=", "permission"}, - {"name", "=", "can_read"}, - {"tail_uuid", "=", group.UUID}, - {"head_kind", "=", "arvados#user"}, - }, - }, &linkList{}) - if err != nil { - return fmt.Errorf("error getting member links: %s", err) + + // Remove previous members not listed on this run + for groupUUID := range remoteGroups { + gi := remoteGroups[groupUUID] + evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers) + groupName := gi.Group.Name + if len(evictedMembers) > 0 { + log.Printf("Removing %d users from group %q", len(evictedMembers), groupName) } - // Build a list of user ids (email or username) belonging to this group - membersSet := make(map[string]bool) - for _, item := range results { - link := item.(link) - memberID, err := allUsers[link.HeadUUID].GetID(*userID) - if err != nil { + for evictedUser := range evictedMembers { + if err := RemoveMemberFromGroup(cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil { return err } - membersSet[memberID] = true + membershipsRemoved++ } - remoteGroups[group.UUID] = &groupInfo{ - Group: group, - PreviousMembers: membersSet, - CurrentMembers: make(map[string]bool), // Empty set - } - groupNameToUUID[group.Name] = group.UUID } - log.Printf("Found %d remote groups", len(remoteGroups)) + log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped) - groupsCreated := 0 - membersAdded := 0 - membersRemoved := 0 + return nil +} +// ProcessFile reads the CSV file and process every record +func ProcessFile( + cfg *ConfigParams, + f *os.File, + userIDToUUID map[string]string, + groupNameToUUID map[string]string, + remoteGroups map[string]*GroupInfo, + allUsers map[string]arvados.User, +) (groupsCreated, membersAdded, membersSkipped int, err error) { + lineNo := 0 csvReader := csv.NewReader(f) + csvReader.FieldsPerRecord = 2 for { - record, err := csvReader.Read() - if err == io.EOF { + record, e := csvReader.Read() + if e == io.EOF { break } - if err != nil { - return fmt.Errorf("error reading %q: %s", *srcPath, err) + lineNo++ + if e != nil { + err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo) + return + } + groupName := strings.TrimSpace(record[0]) + groupMember := strings.TrimSpace(record[1]) // User ID (username or email) + if groupName == "" || groupMember == "" { + log.Printf("Warning: CSV record has at least one empty field (%s, %s). Skipping", groupName, groupMember) + membersSkipped++ + continue } - groupName := record[0] - groupMember := record[1] // User ID (username or email) if _, found := userIDToUUID[groupMember]; !found { // User not present on the system, skip. - log.Printf("Warning: there's no user with %s %q on the system, skipping.", *userID, groupMember) + log.Printf("Warning: there's no user with %s %q on the system, skipping.", cfg.UserID, groupMember) + membersSkipped++ continue } if _, found := groupNameToUUID[groupName]; !found { - // Group doesn't exist, create and tag it before continuing - if *verbose { + // Group doesn't exist, create it before continuing + if cfg.Verbose { log.Printf("Remote group %q not found, creating...", groupName) } - var group group - err := arv.Create("groups", arvadosclient.Dict{ - "group": arvadosclient.Dict{ - "name": groupName, - "owner_uuid": parentGroup.UUID, - }, - }, &group) - if err != nil { - return fmt.Errorf("error creating group named %q: %s", - groupName, err) + var newGroup arvados.Group + groupData := map[string]string{ + "name": groupName, + "owner_uuid": cfg.ParentGroupUUID, + "group_class": "role", } - link := make(map[string]interface{}) - err = arv.Create("links", arvadosclient.Dict{ - "link": arvadosclient.Dict{ - "link_class": "tag", - "name": groupTag, - "head_uuid": group.UUID, - }, - }, &link) - if err != nil { - return fmt.Errorf("error creating tag for group %q: %s", - groupName, err) + if e := CreateGroup(cfg, &newGroup, groupData); e != nil { + err = fmt.Errorf("error creating group named %q: %s", groupName, err) + return } // Update cached group data - groupNameToUUID[groupName] = group.UUID - remoteGroups[group.UUID] = &groupInfo{ - Group: group, + groupNameToUUID[groupName] = newGroup.UUID + remoteGroups[newGroup.UUID] = &GroupInfo{ + Group: newGroup, PreviousMembers: make(map[string]bool), // Empty set CurrentMembers: make(map[string]bool), // Empty set } @@ -418,86 +397,40 @@ func doMain() error { groupUUID := groupNameToUUID[groupName] gi := remoteGroups[groupUUID] if !gi.PreviousMembers[groupMember] && !gi.CurrentMembers[groupMember] { - if *verbose { + if cfg.Verbose { log.Printf("Adding %q to group %q", groupMember, groupName) } - // User wasn't a member, but should. - link := make(map[string]interface{}) - err := arv.Create("links", arvadosclient.Dict{ - "link": arvadosclient.Dict{ - "link_class": "permission", - "name": "can_read", - "tail_uuid": groupUUID, - "head_uuid": userIDToUUID[groupMember], - }, - }, &link) - if err != nil { - return fmt.Errorf("error adding user %q to group %q: %s", - groupMember, groupName, err) + // User wasn't a member, but should be. + if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group); e != nil { + err = e + return } membersAdded++ } gi.CurrentMembers[groupMember] = true } - - // Remove previous members not listed on this run - for groupUUID := range remoteGroups { - gi := remoteGroups[groupUUID] - evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers) - groupName := gi.Group.Name - if len(evictedMembers) > 0 { - log.Printf("Removing %d users from group %q", len(evictedMembers), groupName) - } - for evictedUser := range evictedMembers { - links, err := ListAll(arv, "links", arvadosclient.Dict{ - "filters": [][]string{ - {"link_class", "=", "permission"}, - {"name", "=", "can_read"}, - {"tail_uuid", "=", groupUUID}, - {"head_uuid", "=", userIDToUUID[evictedUser]}, - }, - }, &linkList{}) - if err != nil { - return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", evictedUser, groupName, err) - } - for _, item := range links { - link := item.(link) - var l map[string]interface{} - if *verbose { - log.Printf("Removing %q from group %q", evictedUser, gi.Group.Name) - } - err := arv.Delete("links", link.UUID, arvadosclient.Dict{}, &l) - if err != nil { - return fmt.Errorf("error removing user %q from group %q: %s", evictedUser, groupName, err) - } - } - membersRemoved++ - } - } - log.Printf("Groups created: %d, members added: %d, members removed: %d", groupsCreated, membersAdded, membersRemoved) - - return nil + return } -// ListAll : Adds all objects of type 'resource' to the 'output' list -func ListAll(arv *arvadosclient.ArvadosClient, resource string, parameters arvadosclient.Dict, rl resourceList) (allItems []interface{}, err error) { - if _, ok := parameters["limit"]; !ok { - // Default limit value: use the maximum page size the server allows - parameters["limit"] = 1<<31 - 1 - } - offset := 0 - itemsAvailable := parameters["limit"].(int) - for len(allItems) < itemsAvailable { - parameters["offset"] = offset - err = arv.List(resource, parameters, &rl) - if err != nil { +// GetAll : Adds all objects of type 'resource' to the 'allItems' list +func GetAll(c *arvados.Client, res string, params arvados.ResourceListParams, page resourceList) (allItems []interface{}, err error) { + // Use the maximum page size the server allows + limit := 1<<31 - 1 + params.Limit = &limit + params.Offset = 0 + params.Order = "uuid" + for { + if err = GetResourceList(c, &page, res, params); err != nil { return allItems, err } - for _, i := range rl.items() { + // Have we finished paging? + if page.Len() == 0 { + break + } + for _, i := range page.GetItems() { allItems = append(allItems, i) } - offset = rl.offset() + len(rl.items()) - itemsAvailable = rl.itemsAvailable() + params.Offset += page.Len() } return allItems, nil } @@ -511,3 +444,237 @@ func subtract(setA map[string]bool, setB map[string]bool) map[string]bool { } return result } + +func jsonReader(rscName string, ob interface{}) io.Reader { + j, err := json.Marshal(ob) + if err != nil { + panic(err) + } + v := url.Values{} + v[rscName] = []string{string(j)} + return bytes.NewBufferString(v.Encode()) +} + +// GetRemoteGroups fetches all remote groups with their members +func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remoteGroups map[string]*GroupInfo, groupNameToUUID map[string]string, err error) { + remoteGroups = make(map[string]*GroupInfo) + groupNameToUUID = make(map[string]string) // Index by group name + + params := arvados.ResourceListParams{ + Filters: []arvados.Filter{{ + Attr: "owner_uuid", + Operator: "=", + Operand: cfg.ParentGroupUUID, + }}, + } + results, err := GetAll(cfg.Client, "groups", params, &GroupList{}) + if err != nil { + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote groups: %s", err) + } + for _, item := range results { + group := item.(arvados.Group) + // Group -> User filter + g2uFilter := arvados.ResourceListParams{ + Filters: []arvados.Filter{{ + Attr: "owner_uuid", + Operator: "=", + Operand: cfg.SysUserUUID, + }, { + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "name", + Operator: "=", + Operand: "can_read", + }, { + Attr: "tail_uuid", + Operator: "=", + Operand: group.UUID, + }, { + Attr: "head_kind", + Operator: "=", + Operand: "arvados#user", + }}, + } + // User -> Group filter + u2gFilter := arvados.ResourceListParams{ + Filters: []arvados.Filter{{ + Attr: "owner_uuid", + Operator: "=", + Operand: cfg.SysUserUUID, + }, { + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "name", + Operator: "=", + Operand: "can_write", + }, { + Attr: "head_uuid", + Operator: "=", + Operand: group.UUID, + }, { + Attr: "tail_kind", + Operator: "=", + Operand: "arvados#user", + }}, + } + g2uLinks, err := GetAll(cfg.Client, "links", g2uFilter, &LinkList{}) + if err != nil { + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_read) links for group %q: %s", group.Name, err) + } + u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{}) + if err != nil { + return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_write) links for group %q: %s", group.Name, err) + } + // Build a list of user ids (email or username) belonging to this group + membersSet := make(map[string]bool) + u2gLinkSet := make(map[string]bool) + for _, l := range u2gLinks { + linkedMemberUUID := l.(arvados.Link).TailUUID + u2gLinkSet[linkedMemberUUID] = true + } + for _, item := range g2uLinks { + link := item.(arvados.Link) + // We may have received an old link pointing to a removed account. + if _, found := allUsers[link.HeadUUID]; !found { + continue + } + // The matching User -> Group link may not exist if the link + // creation failed on a previous run. If that's the case, don't + // include this account on the "previous members" list. + if _, found := u2gLinkSet[link.HeadUUID]; !found { + continue + } + memberID, err := GetUserID(allUsers[link.HeadUUID], cfg.UserID) + if err != nil { + return remoteGroups, groupNameToUUID, err + } + membersSet[memberID] = true + } + remoteGroups[group.UUID] = &GroupInfo{ + Group: group, + PreviousMembers: membersSet, + CurrentMembers: make(map[string]bool), // Empty set + } + groupNameToUUID[group.Name] = group.UUID + } + return remoteGroups, groupNameToUUID, nil +} + +// RemoveMemberFromGroup remove all links related to the membership +func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error { + if cfg.Verbose { + log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Email, user.UUID, group.Name, group.UUID) + } + var links []interface{} + // Search for all group<->user links (both ways) + for _, filterset := range [][]arvados.Filter{ + // Group -> User + {{ + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "tail_uuid", + Operator: "=", + Operand: group.UUID, + }, { + Attr: "head_uuid", + Operator: "=", + Operand: user.UUID, + }}, + // Group <- User + {{ + Attr: "link_class", + Operator: "=", + Operand: "permission", + }, { + Attr: "tail_uuid", + Operator: "=", + Operand: user.UUID, + }, { + Attr: "head_uuid", + Operator: "=", + Operand: group.UUID, + }}, + } { + l, err := GetAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &LinkList{}) + if err != nil { + userID, _ := GetUserID(user, cfg.UserID) + return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", userID, group.Name, err) + } + for _, link := range l { + links = append(links, link) + } + } + for _, item := range links { + link := item.(arvados.Link) + userID, _ := GetUserID(user, cfg.UserID) + if cfg.Verbose { + log.Printf("Removing %q permission link for %q on group %q", link.Name, userID, group.Name) + } + if err := DeleteLink(cfg, link.UUID); err != nil { + return fmt.Errorf("error removing user %q from group %q: %s", userID, group.Name, err) + } + } + return nil +} + +// AddMemberToGroup create membership links +func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error { + var newLink arvados.Link + linkData := map[string]string{ + "owner_uuid": cfg.SysUserUUID, + "link_class": "permission", + "name": "can_read", + "tail_uuid": group.UUID, + "head_uuid": user.UUID, + } + if err := CreateLink(cfg, &newLink, linkData); err != nil { + userID, _ := GetUserID(user, cfg.UserID) + return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err) + } + linkData = map[string]string{ + "owner_uuid": cfg.SysUserUUID, + "link_class": "permission", + "name": "can_write", + "tail_uuid": user.UUID, + "head_uuid": group.UUID, + } + if err := CreateLink(cfg, &newLink, linkData); err != nil { + userID, _ := GetUserID(user, cfg.UserID) + return fmt.Errorf("error adding user %q -> group %q write permission: %s", userID, group.Name, err) + } + return nil +} + +// CreateGroup creates a group with groupData parameters, assigns it to dst +func CreateGroup(cfg *ConfigParams, dst *arvados.Group, groupData map[string]string) error { + return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/groups", jsonReader("group", groupData), nil) +} + +// GetGroup fetches a group by its UUID +func GetGroup(cfg *ConfigParams, dst *arvados.Group, groupUUID string) error { + return cfg.Client.RequestAndDecode(&dst, "GET", "/arvados/v1/groups/"+groupUUID, nil, nil) +} + +// CreateLink creates a link with linkData parameters, assigns it to dst +func CreateLink(cfg *ConfigParams, dst *arvados.Link, linkData map[string]string) error { + return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil) +} + +// DeleteLink deletes a link by its UUID +func DeleteLink(cfg *ConfigParams, linkUUID string) error { + if linkUUID == "" { + return fmt.Errorf("cannot delete link with invalid UUID: %q", linkUUID) + } + return cfg.Client.RequestAndDecode(&arvados.Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil) +} + +// GetResourceList fetches res list using params +func GetResourceList(c *arvados.Client, dst *resourceList, res string, params interface{}) error { + return c.RequestAndDecode(dst, "GET", "/arvados/v1/"+res, nil, params) +}