X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a442ad28ac9bcbe782d0e1488a4b38ab0ae7076e..289d2cf581b59632369087388f6163f3979c5e86:/tools/arv-sync-groups/arv-sync-groups.go diff --git a/tools/arv-sync-groups/arv-sync-groups.go b/tools/arv-sync-groups/arv-sync-groups.go index b21d22e653..d7efdefb6f 100644 --- a/tools/arv-sync-groups/arv-sync-groups.go +++ b/tools/arv-sync-groups/arv-sync-groups.go @@ -19,8 +19,6 @@ import ( "git.curoverse.com/arvados.git/sdk/go/arvados" ) -// const remoteGroupParentName string = "Externally synchronized groups" - type resourceList interface { Len() int GetItems() []interface{} @@ -81,21 +79,9 @@ func (l GroupList) GetItems() (out []interface{}) { return } -// Link is an arvados#link record -type Link struct { - UUID string `json:"uuid,omiempty"` - OwnerUUID string `json:"owner_uuid,omitempty"` - Name string `json:"name,omitempty"` - LinkClass string `json:"link_class,omitempty"` - HeadUUID string `json:"head_uuid,omitempty"` - HeadKind string `json:"head_kind,omitempty"` - TailUUID string `json:"tail_uuid,omitempty"` - TailKind string `json:"tail_kind,omitempty"` -} - // LinkList implements resourceList interface type LinkList struct { - Items []Link `json:"items"` + arvados.LinkList } // Len returns the amount of items this list holds @@ -112,7 +98,13 @@ func (l LinkList) GetItems() (out []interface{}) { } func main() { - if err := doMain(); err != nil { + // Parse & validate arguments, set up arvados client. + cfg, err := GetConfig() + if err != nil { + log.Fatalf("%v", err) + } + + if err := doMain(&cfg); err != nil { log.Fatalf("%v", err) } } @@ -135,11 +127,21 @@ func ParseFlags(config *ConfigParams) error { "email": true, // default "username": true, } + flags := flag.NewFlagSet("arv-sync-groups", flag.ExitOnError) - srcPath := flags.String( - "path", - "", - "Local file path containing a CSV format: GroupName,UserID") + + // Set up usage message + flags.Usage = func() { + usageStr := `Synchronize remote groups into Arvados from a CSV format file with 2 columns: + * 1st column: Group name + * 2nd column: User identifier` + fmt.Fprintf(os.Stderr, "%s\n\n", usageStr) + fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] \n\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Options:\n") + flags.PrintDefaults() + } + + // Set up option flags userID := flags.String( "user-id", "email", @@ -156,9 +158,15 @@ func ParseFlags(config *ConfigParams) error { // Parse args; omit the first arg which is the command name flags.Parse(os.Args[1:]) + // Input file as a required positional argument + if flags.NArg() == 0 { + return fmt.Errorf("please provide a path to an input file") + } + srcPath := &os.Args[flags.NFlag()+1] + // Validations if *srcPath == "" { - return fmt.Errorf("please provide a path to an input file") + return fmt.Errorf("input file path invalid") } if !userIDOpts[*userID] { var options []string @@ -260,13 +268,7 @@ func GetConfig() (config ConfigParams, err error) { return config, nil } -func doMain() error { - // Parse & validate arguments, set up arvados client. - cfg, err := GetConfig() - if err != nil { - return err - } - +func doMain(cfg *ConfigParams) error { // Try opening the input file early, just in case there's a problem. f, err := os.Open(cfg.Path) if err != nil { @@ -298,7 +300,7 @@ func doMain() error { } // Get remote groups and their members - remoteGroups, groupNameToUUID, err := GetRemoteGroups(&cfg, allUsers) + remoteGroups, groupNameToUUID, err := GetRemoteGroups(cfg, allUsers) if err != nil { return err } @@ -307,7 +309,7 @@ func doMain() error { membershipsRemoved := 0 // Read the CSV file - groupsCreated, membershipsAdded, membershipsSkipped, err := ProcessFile(&cfg, f, userIDToUUID, groupNameToUUID, remoteGroups, allUsers) + groupsCreated, membershipsAdded, membershipsSkipped, err := ProcessFile(cfg, f, userIDToUUID, groupNameToUUID, remoteGroups, allUsers) if err != nil { return err } @@ -321,7 +323,7 @@ func doMain() error { log.Printf("Removing %d users from group %q", len(evictedMembers), groupName) } for evictedUser := range evictedMembers { - if err := RemoveMemberFromGroup(&cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil { + if err := RemoveMemberFromGroup(cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil { return err } membershipsRemoved++ @@ -333,15 +335,25 @@ func doMain() error { } // ProcessFile reads the CSV file and process every record -func ProcessFile(cfg *ConfigParams, f *os.File, userIDToUUID map[string]string, groupNameToUUID map[string]string, remoteGroups map[string]*GroupInfo, allUsers map[string]arvados.User) (groupsCreated, membersAdded, membersSkipped int, err error) { +func ProcessFile( + cfg *ConfigParams, + f *os.File, + userIDToUUID map[string]string, + groupNameToUUID map[string]string, + remoteGroups map[string]*GroupInfo, + allUsers map[string]arvados.User, +) (groupsCreated, membersAdded, membersSkipped int, err error) { + lineNo := 0 csvReader := csv.NewReader(f) + csvReader.FieldsPerRecord = 2 for { record, e := csvReader.Read() if e == io.EOF { break } + lineNo++ if e != nil { - err = fmt.Errorf("error reading %q: %s", cfg.Path, err) + err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo) return } groupName := strings.TrimSpace(record[0]) @@ -521,11 +533,11 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot membersSet := make(map[string]bool) u2gLinkSet := make(map[string]bool) for _, l := range u2gLinks { - linkedMemberUUID := l.(Link).TailUUID + linkedMemberUUID := l.(arvados.Link).TailUUID u2gLinkSet[linkedMemberUUID] = true } for _, item := range g2uLinks { - link := item.(Link) + link := item.(arvados.Link) // We may have received an old link pointing to a removed account. if _, found := allUsers[link.HeadUUID]; !found { continue @@ -599,12 +611,12 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G } } for _, item := range links { - link := item.(Link) + link := item.(arvados.Link) + userID, _ := GetUserID(user, cfg.UserID) if cfg.Verbose { - log.Printf("Removing %q permission link for %q on group %q", link.Name, user.Email, group.Name) + log.Printf("Removing %q permission link for %q on group %q", link.Name, userID, group.Name) } if err := DeleteLink(cfg, link.UUID); err != nil { - userID, _ := GetUserID(user, cfg.UserID) return fmt.Errorf("error removing user %q from group %q: %s", userID, group.Name, err) } } @@ -613,7 +625,7 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G // AddMemberToGroup create membership links func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error { - var newLink Link + var newLink arvados.Link linkData := map[string]string{ "owner_uuid": cfg.SysUserUUID, "link_class": "permission", @@ -623,7 +635,7 @@ func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) } if err := CreateLink(cfg, &newLink, linkData); err != nil { userID, _ := GetUserID(user, cfg.UserID) - return fmt.Errorf("error adding group %q -> user %q read permission: %s", userID, user.Email, err) + return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err) } linkData = map[string]string{ "owner_uuid": cfg.SysUserUUID, @@ -650,7 +662,7 @@ func GetGroup(cfg *ConfigParams, dst *arvados.Group, groupUUID string) error { } // CreateLink creates a link with linkData parameters, assigns it to dst -func CreateLink(cfg *ConfigParams, dst *Link, linkData map[string]string) error { +func CreateLink(cfg *ConfigParams, dst *arvados.Link, linkData map[string]string) error { return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil) } @@ -659,7 +671,7 @@ func DeleteLink(cfg *ConfigParams, linkUUID string) error { if linkUUID == "" { return fmt.Errorf("cannot delete link with invalid UUID: %q", linkUUID) } - return cfg.Client.RequestAndDecode(&Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil) + return cfg.Client.RequestAndDecode(&arvados.Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil) } // GetResourceList fetches res list using params