1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
19 "git.arvados.org/arvados.git/sdk/go/arvados"
24 type resourceList interface {
26 GetItems() []interface{}
29 // GroupPermissions maps permission levels on groups (can_read, can_write, can_manage)
30 type GroupPermissions map[string]bool
32 // GroupInfo tracks previous and current member's permissions on a particular Group
33 type GroupInfo struct {
35 PreviousMembers map[string]GroupPermissions
36 CurrentMembers map[string]GroupPermissions
39 // GetUserID returns the correct user id value depending on the selector
40 func GetUserID(u arvados.User, idSelector string) (string, error) {
45 return u.Username, nil
47 return "", fmt.Errorf("cannot identify user by %q selector", idSelector)
51 // UserList implements resourceList interface
52 type UserList struct {
56 // Len returns the amount of items this list holds
57 func (l UserList) Len() int {
61 // GetItems returns the list of items
62 func (l UserList) GetItems() (out []interface{}) {
63 for _, item := range l.Items {
64 out = append(out, item)
69 // GroupList implements resourceList interface
70 type GroupList struct {
74 // Len returns the amount of items this list holds
75 func (l GroupList) Len() int {
79 // GetItems returns the list of items
80 func (l GroupList) GetItems() (out []interface{}) {
81 for _, item := range l.Items {
82 out = append(out, item)
87 // LinkList implements resourceList interface
88 type LinkList struct {
92 // Len returns the amount of items this list holds
93 func (l LinkList) Len() int {
97 // GetItems returns the list of items
98 func (l LinkList) GetItems() (out []interface{}) {
99 for _, item := range l.Items {
100 out = append(out, item)
106 // Parse & validate arguments, set up arvados client.
107 cfg, err := GetConfig()
109 log.Fatalf("%v", err)
112 if err := doMain(&cfg); err != nil {
113 log.Fatalf("%v", err)
117 // ConfigParams holds configuration data for this tool
118 type ConfigParams struct {
123 ParentGroupUUID string
124 ParentGroupName string
126 Client *arvados.Client
129 // ParseFlags parses and validates command line arguments
130 func ParseFlags(config *ConfigParams) error {
131 // Acceptable attributes to identify a user on the CSV file
132 userIDOpts := map[string]bool{
133 "email": true, // default
137 flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
139 // Set up usage message
140 flags.Usage = func() {
141 usageStr := `Synchronize remote groups into Arvados from a CSV format file with 3 columns:
143 * 2nd: User identifier
144 * 3rd (Optional): User permission on the group: can_read, can_write or can_manage. (Default: can_write)`
145 fmt.Fprintf(os.Stderr, "%s\n\n", usageStr)
146 fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] <input-file.csv>\n\n", os.Args[0])
147 fmt.Fprintf(os.Stderr, "Options:\n")
148 flags.PrintDefaults()
151 // Set up option flags
152 userID := flags.String(
155 "Attribute by which every user is identified. Valid values are: email and username.")
156 caseInsensitive := flags.Bool(
159 "Performs case insensitive matching on user IDs. Off by default.")
160 verbose := flags.Bool(
163 "Log informational messages. Off by default.")
164 getVersion := flags.Bool(
167 "Print version information and exit.")
168 parentGroupUUID := flags.String(
171 "Use given group UUID as a parent for the remote groups. Should be owned by the system user. If not specified, a group named '"+config.ParentGroupName+"' will be used (and created if nonexistant).")
173 // Parse args; omit the first arg which is the command name
174 flags.Parse(os.Args[1:])
176 // Print version information if requested
178 fmt.Printf("%s %s\n", os.Args[0], version)
182 // Input file as a required positional argument
183 if flags.NArg() == 0 {
184 return fmt.Errorf("please provide a path to an input file")
186 srcPath := &os.Args[flags.NFlag()+1]
190 return fmt.Errorf("input file path invalid")
192 if !userIDOpts[*userID] {
194 for opt := range userIDOpts {
195 options = append(options, opt)
197 return fmt.Errorf("user ID must be one of: %s", strings.Join(options, ", "))
200 config.Path = *srcPath
201 config.ParentGroupUUID = *parentGroupUUID
202 config.UserID = *userID
203 config.Verbose = *verbose
204 config.CaseInsensitive = *caseInsensitive
209 // SetParentGroup finds/create parent group of all remote groups
210 func SetParentGroup(cfg *ConfigParams) error {
211 var parentGroup arvados.Group
212 if cfg.ParentGroupUUID == "" {
213 // UUID not provided, search for preexisting parent group
215 params := arvados.ResourceListParams{
216 Filters: []arvados.Filter{{
219 Operand: cfg.ParentGroupName,
223 Operand: cfg.SysUserUUID,
226 if err := cfg.Client.RequestAndDecode(&gl, "GET", "/arvados/v1/groups", nil, params); err != nil {
227 return fmt.Errorf("error searching for parent group: %s", err)
229 if len(gl.Items) == 0 {
230 // Default parent group does not exist, create it.
232 log.Println("Default parent group not found, creating...")
234 groupData := map[string]string{
235 "name": cfg.ParentGroupName,
236 "owner_uuid": cfg.SysUserUUID,
237 "group_class": "role",
239 if err := CreateGroup(cfg, &parentGroup, groupData); err != nil {
240 return fmt.Errorf("error creating system user owned group named %q: %s", groupData["name"], err)
242 } else if len(gl.Items) == 1 {
243 // Default parent group found.
244 parentGroup = gl.Items[0]
246 // This should never happen, as there's an unique index for
247 // (owner_uuid, name) on groups.
248 return fmt.Errorf("bug: found %d groups owned by system user and named %q", len(gl.Items), cfg.ParentGroupName)
250 cfg.ParentGroupUUID = parentGroup.UUID
252 // UUID provided. Check if exists and if it's owned by system user
253 if err := GetGroup(cfg, &parentGroup, cfg.ParentGroupUUID); err != nil {
254 return fmt.Errorf("error searching for parent group with UUID %q: %s", cfg.ParentGroupUUID, err)
256 if parentGroup.OwnerUUID != cfg.SysUserUUID {
257 return fmt.Errorf("parent group %q (%s) must be owned by system user", parentGroup.Name, cfg.ParentGroupUUID)
263 // GetConfig sets up a ConfigParams struct
264 func GetConfig() (config ConfigParams, err error) {
265 config.ParentGroupName = "Externally synchronized groups"
268 err = ParseFlags(&config)
273 // Arvados Client setup
274 config.Client = arvados.NewClientFromEnv()
276 // Check current user permissions & get System user's UUID
277 u, err := config.Client.CurrentUser()
279 return config, fmt.Errorf("error getting the current user: %s", err)
281 if !u.IsActive || !u.IsAdmin {
282 return config, fmt.Errorf("current user (%s) is not an active admin user", u.UUID)
285 var ac struct{ ClusterID string }
286 err = config.Client.RequestAndDecode(&ac, "GET", "arvados/v1/config", nil, nil)
288 return config, fmt.Errorf("error getting the exported config: %s", err)
290 config.SysUserUUID = ac.ClusterID + "-tpzed-000000000000000"
292 // Set up remote groups' parent
293 if err = SetParentGroup(&config); err != nil {
300 func doMain(cfg *ConfigParams) error {
301 // Try opening the input file early, just in case there's a problem.
302 f, err := os.Open(cfg.Path)
304 return fmt.Errorf("%s", err)
308 log.Printf("%s %s started. Using %q as users id and parent group UUID %q", os.Args[0], version, cfg.UserID, cfg.ParentGroupUUID)
310 // Get the complete user list to minimize API Server requests
311 allUsers := make(map[string]arvados.User)
312 userIDToUUID := make(map[string]string) // Index by email or username
313 results, err := GetAll(cfg.Client, "users", arvados.ResourceListParams{}, &UserList{})
315 return fmt.Errorf("error getting user list: %s", err)
317 log.Printf("Found %d users", len(results))
318 for _, item := range results {
319 u := item.(arvados.User)
321 uID, err := GetUserID(u, cfg.UserID)
325 userIDToUUID[uID] = u.UUID
327 log.Printf("Seen user %q (%s)", u.Username, u.UUID)
331 // Get remote groups and their members
332 remoteGroups, groupNameToUUID, err := GetRemoteGroups(cfg, allUsers)
336 log.Printf("Found %d remote groups", len(remoteGroups))
338 for groupUUID := range remoteGroups {
339 log.Printf("- Group %q: %d users", remoteGroups[groupUUID].Group.Name, len(remoteGroups[groupUUID].PreviousMembers))
343 membershipsRemoved := 0
346 groupsCreated, membershipsAdded, membershipsSkipped, err := ProcessFile(cfg, f, userIDToUUID, groupNameToUUID, remoteGroups, allUsers)
351 // Remove previous members not listed on this run
352 for groupUUID := range remoteGroups {
353 gi := remoteGroups[groupUUID]
354 evictedMemberPerms := subtract(gi.PreviousMembers, gi.CurrentMembers)
355 groupName := gi.Group.Name
356 if len(evictedMemberPerms) > 0 {
357 log.Printf("Removing permissions from %d users on group %q", len(evictedMemberPerms), groupName)
359 for member := range evictedMemberPerms {
361 completeMembershipRemoval := false
362 if _, ok := gi.CurrentMembers[member]; !ok {
363 completeMembershipRemoval = true
366 // Collect which user->group permission links should be removed
367 for p := range evictedMemberPerms[member] {
368 if evictedMemberPerms[member][p] {
369 perms = append(perms, p)
372 membershipsRemoved += len(perms)
374 if err := RemoveMemberLinksFromGroup(cfg, allUsers[userIDToUUID[member]],
375 perms, completeMembershipRemoval, gi.Group); err != nil {
380 log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped)
385 // ProcessFile reads the CSV file and process every record
389 userIDToUUID map[string]string,
390 groupNameToUUID map[string]string,
391 remoteGroups map[string]*GroupInfo,
392 allUsers map[string]arvados.User,
393 ) (groupsCreated, membersAdded, membersSkipped int, err error) {
395 csvReader := csv.NewReader(f)
396 // Allow variable number of fields.
397 csvReader.FieldsPerRecord = -1
399 record, e := csvReader.Read()
405 err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo)
408 // Only allow 2 or 3 fields per record for backwards compatibility.
409 if len(record) < 2 || len(record) > 3 {
410 err = fmt.Errorf("error parsing %q, line %d: found %d fields but only 2 or 3 are allowed", cfg.Path, lineNo, len(record))
413 groupName := strings.TrimSpace(record[0])
414 groupMember := strings.TrimSpace(record[1]) // User ID (username or email)
415 groupPermission := "can_write"
416 if len(record) == 3 {
417 groupPermission = strings.ToLower(record[2])
419 if groupName == "" || groupMember == "" || groupPermission == "" {
420 log.Printf("Warning: CSV record has at least one empty field (%s, %s, %s). Skipping", groupName, groupMember, groupPermission)
424 if !(groupPermission == "can_read" || groupPermission == "can_write" || groupPermission == "can_manage") {
425 log.Printf("Warning: 3rd field should be 'can_read', 'can_write' or 'can_manage'. Found: %q at line %d, skipping.", groupPermission, lineNo)
429 if _, found := userIDToUUID[groupMember]; !found {
430 // User not present on the system, skip.
431 log.Printf("Warning: there's no user with %s %q on the system, skipping.", cfg.UserID, groupMember)
435 if _, found := groupNameToUUID[groupName]; !found {
436 // Group doesn't exist, create it before continuing
438 log.Printf("Remote group %q not found, creating...", groupName)
440 var newGroup arvados.Group
441 groupData := map[string]string{
443 "owner_uuid": cfg.ParentGroupUUID,
444 "group_class": "role",
446 if e := CreateGroup(cfg, &newGroup, groupData); e != nil {
447 err = fmt.Errorf("error creating group named %q: %s", groupName, e)
450 // Update cached group data
451 groupNameToUUID[groupName] = newGroup.UUID
452 remoteGroups[newGroup.UUID] = &GroupInfo{
454 PreviousMembers: make(map[string]GroupPermissions),
455 CurrentMembers: make(map[string]GroupPermissions),
459 // Both group & user exist, check if user is a member
460 groupUUID := groupNameToUUID[groupName]
461 gi := remoteGroups[groupUUID]
462 if !gi.PreviousMembers[groupMember][groupPermission] && !gi.CurrentMembers[groupMember][groupPermission] {
464 log.Printf("Adding %q to group %q", groupMember, groupName)
466 // User permissionwasn't there, but should be. Avoid duplicating the
467 // group->user link when necessary.
468 createG2ULink := true
469 if _, ok := gi.PreviousMembers[groupMember]; ok {
470 createG2ULink = false // User is already member of the group
472 if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group, groupPermission, createG2ULink); e != nil {
478 if _, ok := gi.CurrentMembers[groupMember]; ok {
479 gi.CurrentMembers[groupMember][groupPermission] = true
481 gi.CurrentMembers[groupMember] = GroupPermissions{groupPermission: true}
488 // GetAll : Adds all objects of type 'resource' to the 'allItems' list
489 func GetAll(c *arvados.Client, res string, params arvados.ResourceListParams, page resourceList) (allItems []interface{}, err error) {
490 // Use the maximum page size the server allows
492 params.Limit = &limit
494 params.Order = "uuid"
496 if err = GetResourceList(c, &page, res, params); err != nil {
499 // Have we finished paging?
503 allItems = append(allItems, page.GetItems()...)
504 params.Offset += page.Len()
509 func subtract(setA map[string]GroupPermissions, setB map[string]GroupPermissions) map[string]GroupPermissions {
510 result := make(map[string]GroupPermissions)
511 for element := range setA {
512 if _, ok := setB[element]; !ok {
513 result[element] = setA[element]
515 for perm := range setA[element] {
516 if _, ok := setB[element][perm]; !ok {
517 result[element] = GroupPermissions{perm: true}
525 func jsonReader(rscName string, ob interface{}) io.Reader {
526 j, err := json.Marshal(ob)
531 v[rscName] = []string{string(j)}
532 return bytes.NewBufferString(v.Encode())
535 // GetRemoteGroups fetches all remote groups with their members
536 func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remoteGroups map[string]*GroupInfo, groupNameToUUID map[string]string, err error) {
537 remoteGroups = make(map[string]*GroupInfo)
538 groupNameToUUID = make(map[string]string) // Index by group name
540 params := arvados.ResourceListParams{
541 Filters: []arvados.Filter{{
544 Operand: cfg.ParentGroupUUID,
547 results, err := GetAll(cfg.Client, "links", params, &LinkList{})
549 return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote groups: %s", err)
551 for _, item := range results {
552 var group arvados.Group
553 err = GetGroup(cfg, &group, item.(arvados.Link).HeadUUID)
555 return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote group: %s", err)
557 // Group -> User filter
558 g2uFilter := arvados.ResourceListParams{
559 Filters: []arvados.Filter{{
562 Operand: cfg.SysUserUUID,
566 Operand: "permission",
578 Operand: "arvados#user",
581 // User -> Group filter
582 u2gFilter := arvados.ResourceListParams{
583 Filters: []arvados.Filter{{
586 Operand: cfg.SysUserUUID,
590 Operand: "permission",
594 Operand: []string{"can_read", "can_write", "can_manage"},
602 Operand: "arvados#user",
605 g2uLinks, err := GetAll(cfg.Client, "links", g2uFilter, &LinkList{})
607 return remoteGroups, groupNameToUUID, fmt.Errorf("error getting group->user 'can_read' links for group %q: %s", group.Name, err)
609 u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{})
611 return remoteGroups, groupNameToUUID, fmt.Errorf("error getting user->group links for group %q: %s", group.Name, err)
613 // Build a list of user ids (email or username) belonging to this group.
614 membersSet := make(map[string]GroupPermissions)
615 u2gLinkSet := make(map[string]GroupPermissions)
616 for _, l := range u2gLinks {
617 link := l.(arvados.Link)
618 // Also save the member's group access level.
619 if _, ok := u2gLinkSet[link.TailUUID]; ok {
620 u2gLinkSet[link.TailUUID][link.Name] = true
622 u2gLinkSet[link.TailUUID] = GroupPermissions{link.Name: true}
625 for _, item := range g2uLinks {
626 link := item.(arvados.Link)
627 // We may have received an old link pointing to a removed account.
628 if _, found := allUsers[link.HeadUUID]; !found {
631 // The matching User -> Group link may not exist if the link
632 // creation failed on a previous run. If that's the case, don't
633 // include this account on the "previous members" list.
634 if _, found := u2gLinkSet[link.HeadUUID]; !found {
637 memberID, err := GetUserID(allUsers[link.HeadUUID], cfg.UserID)
639 return remoteGroups, groupNameToUUID, err
641 membersSet[memberID] = u2gLinkSet[link.HeadUUID]
643 remoteGroups[group.UUID] = &GroupInfo{
645 PreviousMembers: membersSet,
646 CurrentMembers: make(map[string]GroupPermissions),
648 groupNameToUUID[group.Name] = group.UUID
650 return remoteGroups, groupNameToUUID, nil
653 // RemoveMemberLinksFromGroup remove all links related to the membership
654 func RemoveMemberLinksFromGroup(cfg *ConfigParams, user arvados.User, linkNames []string, completeRemoval bool, group arvados.Group) error {
656 log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Username, user.UUID, group.Name, group.UUID)
658 var links []interface{}
659 var filters [][]arvados.Filter
661 // Search for all group<->user links (both ways)
662 filters = [][]arvados.Filter{
667 Operand: "permission",
681 Operand: "permission",
693 // Search only for the requested Group <- User permission links
694 filters = [][]arvados.Filter{
698 Operand: "permission",
715 for _, filterset := range filters {
716 l, err := GetAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &LinkList{})
718 userID, _ := GetUserID(user, cfg.UserID)
719 return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", userID, group.Name, err)
721 links = append(links, l...)
723 for _, item := range links {
724 link := item.(arvados.Link)
725 userID, _ := GetUserID(user, cfg.UserID)
727 log.Printf("Removing %q permission link for %q on group %q", link.Name, userID, group.Name)
729 if err := DeleteLink(cfg, link.UUID); err != nil {
730 return fmt.Errorf("error removing user %q from group %q: %s", userID, group.Name, err)
736 // AddMemberToGroup create membership links
737 func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group, perm string, createG2ULink bool) error {
738 var newLink arvados.Link
739 var linkData map[string]string
741 linkData = map[string]string{
742 "owner_uuid": cfg.SysUserUUID,
743 "link_class": "permission",
745 "tail_uuid": group.UUID,
746 "head_uuid": user.UUID,
748 if err := CreateLink(cfg, &newLink, linkData); err != nil {
749 userID, _ := GetUserID(user, cfg.UserID)
750 return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err)
753 linkData = map[string]string{
754 "owner_uuid": cfg.SysUserUUID,
755 "link_class": "permission",
757 "tail_uuid": user.UUID,
758 "head_uuid": group.UUID,
760 if err := CreateLink(cfg, &newLink, linkData); err != nil {
761 userID, _ := GetUserID(user, cfg.UserID)
762 return fmt.Errorf("error adding user %q -> group %q %s permission: %s", userID, group.Name, perm, err)
767 // CreateGroup creates a group with groupData parameters, assigns it to dst
768 func CreateGroup(cfg *ConfigParams, dst *arvados.Group, groupData map[string]string) error {
769 return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/groups", jsonReader("group", groupData), nil)
772 // GetGroup fetches a group by its UUID
773 func GetGroup(cfg *ConfigParams, dst *arvados.Group, groupUUID string) error {
774 return cfg.Client.RequestAndDecode(&dst, "GET", "/arvados/v1/groups/"+groupUUID, nil, nil)
777 // CreateLink creates a link with linkData parameters, assigns it to dst
778 func CreateLink(cfg *ConfigParams, dst *arvados.Link, linkData map[string]string) error {
779 return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil)
782 // DeleteLink deletes a link by its UUID
783 func DeleteLink(cfg *ConfigParams, linkUUID string) error {
785 return fmt.Errorf("cannot delete link with invalid UUID: %q", linkUUID)
787 return cfg.Client.RequestAndDecode(&arvados.Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil)
790 // GetResourceList fetches res list using params
791 func GetResourceList(c *arvados.Client, dst *resourceList, res string, params interface{}) error {
792 return c.RequestAndDecode(dst, "GET", "/arvados/v1/"+res, nil, params)