"git.curoverse.com/arvados.git/sdk/go/arvados"
)
-// const remoteGroupParentName string = "Externally synchronized groups"
+var version = "dev"
type resourceList interface {
Len() int
return
}
-// Link is an arvados#link record
-type Link struct {
- UUID string `json:"uuid,omiempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
- Name string `json:"name,omitempty"`
- LinkClass string `json:"link_class,omitempty"`
- HeadUUID string `json:"head_uuid,omitempty"`
- HeadKind string `json:"head_kind,omitempty"`
- TailUUID string `json:"tail_uuid,omitempty"`
- TailKind string `json:"tail_kind,omitempty"`
-}
-
// LinkList implements resourceList interface
type LinkList struct {
- Items []Link `json:"items"`
+ arvados.LinkList
}
// Len returns the amount of items this list holds
}
func main() {
- if err := doMain(); err != nil {
+ // Parse & validate arguments, set up arvados client.
+ cfg, err := GetConfig()
+ if err != nil {
+ log.Fatalf("%v", err)
+ }
+
+ if err := doMain(&cfg); err != nil {
log.Fatalf("%v", err)
}
}
"email": true, // default
"username": true,
}
+
flags := flag.NewFlagSet("arv-sync-groups", flag.ExitOnError)
- srcPath := flags.String(
- "path",
- "",
- "Local file path containing a CSV format: GroupName,UserID")
+
+ // Set up usage message
+ flags.Usage = func() {
+ usageStr := `Synchronize remote groups into Arvados from a CSV format file with 2 columns:
+ * 1st column: Group name
+ * 2nd column: User identifier`
+ fmt.Fprintf(os.Stderr, "%s\n\n", usageStr)
+ fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] <input-file.csv>\n\n", os.Args[0])
+ fmt.Fprintf(os.Stderr, "Options:\n")
+ flags.PrintDefaults()
+ }
+
+ // Set up option flags
userID := flags.String(
"user-id",
"email",
"verbose",
false,
"Log informational messages. Off by default.")
+ getVersion := flags.Bool(
+ "version",
+ false,
+ "Print version information and exit.")
parentGroupUUID := flags.String(
"parent-group-uuid",
"",
// Parse args; omit the first arg which is the command name
flags.Parse(os.Args[1:])
+ // Print version information if requested
+ if *getVersion {
+ fmt.Printf("arv-sync-groups %s\n", version)
+ os.Exit(0)
+ }
+
+ // Input file as a required positional argument
+ if flags.NArg() == 0 {
+ return fmt.Errorf("please provide a path to an input file")
+ }
+ srcPath := &os.Args[flags.NFlag()+1]
+
// Validations
if *srcPath == "" {
- return fmt.Errorf("please provide a path to an input file")
+ return fmt.Errorf("input file path invalid")
}
if !userIDOpts[*userID] {
var options []string
return config, nil
}
-func doMain() error {
- // Parse & validate arguments, set up arvados client.
- cfg, err := GetConfig()
- if err != nil {
- return err
- }
-
+func doMain(cfg *ConfigParams) error {
// Try opening the input file early, just in case there's a problem.
f, err := os.Open(cfg.Path)
if err != nil {
}
defer f.Close()
- log.Printf("Group sync starting. Using %q as users id and parent group UUID %q", cfg.UserID, cfg.ParentGroupUUID)
+ log.Printf("arv-sync-groups %s started. Using %q as users id and parent group UUID %q", version, cfg.UserID, cfg.ParentGroupUUID)
// Get the complete user list to minimize API Server requests
allUsers := make(map[string]arvados.User)
}
// Get remote groups and their members
- remoteGroups, groupNameToUUID, err := GetRemoteGroups(&cfg, allUsers)
+ remoteGroups, groupNameToUUID, err := GetRemoteGroups(cfg, allUsers)
if err != nil {
return err
}
membershipsRemoved := 0
// Read the CSV file
- groupsCreated, membershipsAdded, membershipsSkipped, err := ProcessFile(&cfg, f, userIDToUUID, groupNameToUUID, remoteGroups, allUsers)
+ groupsCreated, membershipsAdded, membershipsSkipped, err := ProcessFile(cfg, f, userIDToUUID, groupNameToUUID, remoteGroups, allUsers)
if err != nil {
return err
}
log.Printf("Removing %d users from group %q", len(evictedMembers), groupName)
}
for evictedUser := range evictedMembers {
- if err := RemoveMemberFromGroup(&cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil {
+ if err := RemoveMemberFromGroup(cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil {
return err
}
membershipsRemoved++
}
// ProcessFile reads the CSV file and process every record
-func ProcessFile(cfg *ConfigParams, f *os.File, userIDToUUID map[string]string, groupNameToUUID map[string]string, remoteGroups map[string]*GroupInfo, allUsers map[string]arvados.User) (groupsCreated, membersAdded, membersSkipped int, err error) {
+func ProcessFile(
+ cfg *ConfigParams,
+ f *os.File,
+ userIDToUUID map[string]string,
+ groupNameToUUID map[string]string,
+ remoteGroups map[string]*GroupInfo,
+ allUsers map[string]arvados.User,
+) (groupsCreated, membersAdded, membersSkipped int, err error) {
+ lineNo := 0
csvReader := csv.NewReader(f)
+ csvReader.FieldsPerRecord = 2
for {
record, e := csvReader.Read()
if e == io.EOF {
break
}
+ lineNo++
if e != nil {
- err = fmt.Errorf("error reading %q: %s", cfg.Path, err)
+ err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo)
return
}
groupName := strings.TrimSpace(record[0])
}
var newGroup arvados.Group
groupData := map[string]string{
- "name": groupName,
- "owner_uuid": cfg.ParentGroupUUID,
+ "name": groupName,
+ "owner_uuid": cfg.ParentGroupUUID,
+ "group_class": "role",
}
if e := CreateGroup(cfg, &newGroup, groupData); e != nil {
err = fmt.Errorf("error creating group named %q: %s", groupName, err)
membersSet := make(map[string]bool)
u2gLinkSet := make(map[string]bool)
for _, l := range u2gLinks {
- linkedMemberUUID := l.(Link).TailUUID
+ linkedMemberUUID := l.(arvados.Link).TailUUID
u2gLinkSet[linkedMemberUUID] = true
}
for _, item := range g2uLinks {
- link := item.(Link)
+ link := item.(arvados.Link)
// We may have received an old link pointing to a removed account.
if _, found := allUsers[link.HeadUUID]; !found {
continue
}
}
for _, item := range links {
- link := item.(Link)
+ link := item.(arvados.Link)
+ userID, _ := GetUserID(user, cfg.UserID)
if cfg.Verbose {
- log.Printf("Removing %q permission link for %q on group %q", link.Name, user.Email, group.Name)
+ log.Printf("Removing %q permission link for %q on group %q", link.Name, userID, group.Name)
}
if err := DeleteLink(cfg, link.UUID); err != nil {
- userID, _ := GetUserID(user, cfg.UserID)
return fmt.Errorf("error removing user %q from group %q: %s", userID, group.Name, err)
}
}
// AddMemberToGroup create membership links
func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
- var newLink Link
+ var newLink arvados.Link
linkData := map[string]string{
"owner_uuid": cfg.SysUserUUID,
"link_class": "permission",
}
if err := CreateLink(cfg, &newLink, linkData); err != nil {
userID, _ := GetUserID(user, cfg.UserID)
- return fmt.Errorf("error adding group %q -> user %q read permission: %s", userID, user.Email, err)
+ return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err)
}
linkData = map[string]string{
"owner_uuid": cfg.SysUserUUID,
}
// CreateLink creates a link with linkData parameters, assigns it to dst
-func CreateLink(cfg *ConfigParams, dst *Link, linkData map[string]string) error {
+func CreateLink(cfg *ConfigParams, dst *arvados.Link, linkData map[string]string) error {
return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil)
}
if linkUUID == "" {
return fmt.Errorf("cannot delete link with invalid UUID: %q", linkUUID)
}
- return cfg.Client.RequestAndDecode(&Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil)
+ return cfg.Client.RequestAndDecode(&arvados.Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil)
}
// GetResourceList fetches res list using params
projects / arvados.git / blobdiff