--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "bytes"
+ "encoding/csv"
+ "encoding/json"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "net/url"
+ "os"
+ "strings"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+)
+
+type resourceList interface {
+ Len() int
+ GetItems() []interface{}
+}
+
+// GroupInfo tracks previous and current members of a particular Group
+type GroupInfo struct {
+ Group arvados.Group
+ PreviousMembers map[string]bool
+ CurrentMembers map[string]bool
+}
+
+// GetUserID returns the correct user id value depending on the selector
+func GetUserID(u arvados.User, idSelector string) (string, error) {
+ switch idSelector {
+ case "email":
+ return u.Email, nil
+ case "username":
+ return u.Username, nil
+ default:
+ return "", fmt.Errorf("cannot identify user by %q selector", idSelector)
+ }
+}
+
+// UserList implements resourceList interface
+type UserList struct {
+ arvados.UserList
+}
+
+// Len returns the amount of items this list holds
+func (l UserList) Len() int {
+ return len(l.Items)
+}
+
+// GetItems returns the list of items
+func (l UserList) GetItems() (out []interface{}) {
+ for _, item := range l.Items {
+ out = append(out, item)
+ }
+ return
+}
+
+// GroupList implements resourceList interface
+type GroupList struct {
+ arvados.GroupList
+}
+
+// Len returns the amount of items this list holds
+func (l GroupList) Len() int {
+ return len(l.Items)
+}
+
+// GetItems returns the list of items
+func (l GroupList) GetItems() (out []interface{}) {
+ for _, item := range l.Items {
+ out = append(out, item)
+ }
+ return
+}
+
+// LinkList implements resourceList interface
+type LinkList struct {
+ arvados.LinkList
+}
+
+// Len returns the amount of items this list holds
+func (l LinkList) Len() int {
+ return len(l.Items)
+}
+
+// GetItems returns the list of items
+func (l LinkList) GetItems() (out []interface{}) {
+ for _, item := range l.Items {
+ out = append(out, item)
+ }
+ return
+}
+
+func main() {
+ // Parse & validate arguments, set up arvados client.
+ cfg, err := GetConfig()
+ if err != nil {
+ log.Fatalf("%v", err)
+ }
+
+ if err := doMain(&cfg); err != nil {
+ log.Fatalf("%v", err)
+ }
+}
+
+// ConfigParams holds configuration data for this tool
+type ConfigParams struct {
+ Path string
+ UserID string
+ Verbose bool
+ ParentGroupUUID string
+ ParentGroupName string
+ SysUserUUID string
+ Client *arvados.Client
+}
+
+// ParseFlags parses and validates command line arguments
+func ParseFlags(config *ConfigParams) error {
+ // Acceptable attributes to identify a user on the CSV file
+ userIDOpts := map[string]bool{
+ "email": true, // default
+ "username": true,
+ }
+
+ flags := flag.NewFlagSet("arv-sync-groups", flag.ExitOnError)
+
+ // Set up usage message
+ flags.Usage = func() {
+ usageStr := `Synchronize remote groups into Arvados from a CSV format file with 2 columns:
+ * 1st column: Group name
+ * 2nd column: User identifier`
+ fmt.Fprintf(os.Stderr, "%s\n\n", usageStr)
+ fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] <input-file.csv>\n\n", os.Args[0])
+ fmt.Fprintf(os.Stderr, "Options:\n")
+ flags.PrintDefaults()
+ }
+
+ // Set up option flags
+ userID := flags.String(
+ "user-id",
+ "email",
+ "Attribute by which every user is identified. Valid values are: email and username.")
+ verbose := flags.Bool(
+ "verbose",
+ false,
+ "Log informational messages. Off by default.")
+ parentGroupUUID := flags.String(
+ "parent-group-uuid",
+ "",
+ "Use given group UUID as a parent for the remote groups. Should be owned by the system user. If not specified, a group named '"+config.ParentGroupName+"' will be used (and created if nonexistant).")
+
+ // Parse args; omit the first arg which is the command name
+ flags.Parse(os.Args[1:])
+
+ // Input file as a required positional argument
+ if flags.NArg() == 0 {
+ return fmt.Errorf("please provide a path to an input file")
+ }
+ srcPath := &os.Args[flags.NFlag()+1]
+
+ // Validations
+ if *srcPath == "" {
+ return fmt.Errorf("input file path invalid")
+ }
+ if !userIDOpts[*userID] {
+ var options []string
+ for opt := range userIDOpts {
+ options = append(options, opt)
+ }
+ return fmt.Errorf("user ID must be one of: %s", strings.Join(options, ", "))
+ }
+
+ config.Path = *srcPath
+ config.ParentGroupUUID = *parentGroupUUID
+ config.UserID = *userID
+ config.Verbose = *verbose
+
+ return nil
+}
+
+// SetParentGroup finds/create parent group of all remote groups
+func SetParentGroup(cfg *ConfigParams) error {
+ var parentGroup arvados.Group
+ if cfg.ParentGroupUUID == "" {
+ // UUID not provided, search for preexisting parent group
+ var gl GroupList
+ params := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "name",
+ Operator: "=",
+ Operand: cfg.ParentGroupName,
+ }, {
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: cfg.SysUserUUID,
+ }},
+ }
+ if err := cfg.Client.RequestAndDecode(&gl, "GET", "/arvados/v1/groups", nil, params); err != nil {
+ return fmt.Errorf("error searching for parent group: %s", err)
+ }
+ if len(gl.Items) == 0 {
+ // Default parent group not existant, create one.
+ if cfg.Verbose {
+ log.Println("Default parent group not found, creating...")
+ }
+ groupData := map[string]string{
+ "name": cfg.ParentGroupName,
+ "owner_uuid": cfg.SysUserUUID,
+ }
+ if err := CreateGroup(cfg, &parentGroup, groupData); err != nil {
+ return fmt.Errorf("error creating system user owned group named %q: %s", groupData["name"], err)
+ }
+ } else if len(gl.Items) == 1 {
+ // Default parent group found.
+ parentGroup = gl.Items[0]
+ } else {
+ // This should never happen, as there's an unique index for
+ // (owner_uuid, name) on groups.
+ return fmt.Errorf("bug: found %d groups owned by system user and named %q", len(gl.Items), cfg.ParentGroupName)
+ }
+ cfg.ParentGroupUUID = parentGroup.UUID
+ } else {
+ // UUID provided. Check if exists and if it's owned by system user
+ if err := GetGroup(cfg, &parentGroup, cfg.ParentGroupUUID); err != nil {
+ return fmt.Errorf("error searching for parent group with UUID %q: %s", cfg.ParentGroupUUID, err)
+ }
+ if parentGroup.OwnerUUID != cfg.SysUserUUID {
+ return fmt.Errorf("parent group %q (%s) must be owned by system user", parentGroup.Name, cfg.ParentGroupUUID)
+ }
+ }
+ return nil
+}
+
+// GetConfig sets up a ConfigParams struct
+func GetConfig() (config ConfigParams, err error) {
+ config.ParentGroupName = "Externally synchronized groups"
+
+ // Command arguments
+ err = ParseFlags(&config)
+ if err != nil {
+ return config, err
+ }
+
+ // Arvados Client setup
+ config.Client = arvados.NewClientFromEnv()
+
+ // Check current user permissions & get System user's UUID
+ u, err := config.Client.CurrentUser()
+ if err != nil {
+ return config, fmt.Errorf("error getting the current user: %s", err)
+ }
+ if !u.IsActive || !u.IsAdmin {
+ return config, fmt.Errorf("current user (%s) is not an active admin user", u.UUID)
+ }
+ config.SysUserUUID = u.UUID[:12] + "000000000000000"
+
+ // Set up remote groups' parent
+ if err = SetParentGroup(&config); err != nil {
+ return config, err
+ }
+
+ return config, nil
+}
+
+func doMain(cfg *ConfigParams) error {
+ // Try opening the input file early, just in case there's a problem.
+ f, err := os.Open(cfg.Path)
+ if err != nil {
+ return fmt.Errorf("%s", err)
+ }
+ defer f.Close()
+
+ log.Printf("Group sync starting. Using %q as users id and parent group UUID %q", cfg.UserID, cfg.ParentGroupUUID)
+
+ // Get the complete user list to minimize API Server requests
+ allUsers := make(map[string]arvados.User)
+ userIDToUUID := make(map[string]string) // Index by email or username
+ results, err := GetAll(cfg.Client, "users", arvados.ResourceListParams{}, &UserList{})
+ if err != nil {
+ return fmt.Errorf("error getting user list: %s", err)
+ }
+ log.Printf("Found %d users", len(results))
+ for _, item := range results {
+ u := item.(arvados.User)
+ allUsers[u.UUID] = u
+ uID, err := GetUserID(u, cfg.UserID)
+ if err != nil {
+ return err
+ }
+ userIDToUUID[uID] = u.UUID
+ if cfg.Verbose {
+ log.Printf("Seen user %q (%s)", u.Username, u.Email)
+ }
+ }
+
+ // Get remote groups and their members
+ remoteGroups, groupNameToUUID, err := GetRemoteGroups(cfg, allUsers)
+ if err != nil {
+ return err
+ }
+ log.Printf("Found %d remote groups", len(remoteGroups))
+
+ membershipsRemoved := 0
+
+ // Read the CSV file
+ groupsCreated, membershipsAdded, membershipsSkipped, err := ProcessFile(cfg, f, userIDToUUID, groupNameToUUID, remoteGroups, allUsers)
+ if err != nil {
+ return err
+ }
+
+ // Remove previous members not listed on this run
+ for groupUUID := range remoteGroups {
+ gi := remoteGroups[groupUUID]
+ evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers)
+ groupName := gi.Group.Name
+ if len(evictedMembers) > 0 {
+ log.Printf("Removing %d users from group %q", len(evictedMembers), groupName)
+ }
+ for evictedUser := range evictedMembers {
+ if err := RemoveMemberFromGroup(cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil {
+ return err
+ }
+ membershipsRemoved++
+ }
+ }
+ log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped)
+
+ return nil
+}
+
+// ProcessFile reads the CSV file and process every record
+func ProcessFile(
+ cfg *ConfigParams,
+ f *os.File,
+ userIDToUUID map[string]string,
+ groupNameToUUID map[string]string,
+ remoteGroups map[string]*GroupInfo,
+ allUsers map[string]arvados.User,
+) (groupsCreated, membersAdded, membersSkipped int, err error) {
+ lineNo := 0
+ csvReader := csv.NewReader(f)
+ csvReader.FieldsPerRecord = 2
+ for {
+ record, e := csvReader.Read()
+ if e == io.EOF {
+ break
+ }
+ lineNo++
+ if e != nil {
+ err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo)
+ return
+ }
+ groupName := strings.TrimSpace(record[0])
+ groupMember := strings.TrimSpace(record[1]) // User ID (username or email)
+ if groupName == "" || groupMember == "" {
+ log.Printf("Warning: CSV record has at least one empty field (%s, %s). Skipping", groupName, groupMember)
+ membersSkipped++
+ continue
+ }
+ if _, found := userIDToUUID[groupMember]; !found {
+ // User not present on the system, skip.
+ log.Printf("Warning: there's no user with %s %q on the system, skipping.", cfg.UserID, groupMember)
+ membersSkipped++
+ continue
+ }
+ if _, found := groupNameToUUID[groupName]; !found {
+ // Group doesn't exist, create it before continuing
+ if cfg.Verbose {
+ log.Printf("Remote group %q not found, creating...", groupName)
+ }
+ var newGroup arvados.Group
+ groupData := map[string]string{
+ "name": groupName,
+ "owner_uuid": cfg.ParentGroupUUID,
+ "group_class": "role",
+ }
+ if e := CreateGroup(cfg, &newGroup, groupData); e != nil {
+ err = fmt.Errorf("error creating group named %q: %s", groupName, err)
+ return
+ }
+ // Update cached group data
+ groupNameToUUID[groupName] = newGroup.UUID
+ remoteGroups[newGroup.UUID] = &GroupInfo{
+ Group: newGroup,
+ PreviousMembers: make(map[string]bool), // Empty set
+ CurrentMembers: make(map[string]bool), // Empty set
+ }
+ groupsCreated++
+ }
+ // Both group & user exist, check if user is a member
+ groupUUID := groupNameToUUID[groupName]
+ gi := remoteGroups[groupUUID]
+ if !gi.PreviousMembers[groupMember] && !gi.CurrentMembers[groupMember] {
+ if cfg.Verbose {
+ log.Printf("Adding %q to group %q", groupMember, groupName)
+ }
+ // User wasn't a member, but should be.
+ if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group); e != nil {
+ err = e
+ return
+ }
+ membersAdded++
+ }
+ gi.CurrentMembers[groupMember] = true
+ }
+ return
+}
+
+// GetAll : Adds all objects of type 'resource' to the 'allItems' list
+func GetAll(c *arvados.Client, res string, params arvados.ResourceListParams, page resourceList) (allItems []interface{}, err error) {
+ // Use the maximum page size the server allows
+ limit := 1<<31 - 1
+ params.Limit = &limit
+ params.Offset = 0
+ params.Order = "uuid"
+ for {
+ if err = GetResourceList(c, &page, res, params); err != nil {
+ return allItems, err
+ }
+ // Have we finished paging?
+ if page.Len() == 0 {
+ break
+ }
+ for _, i := range page.GetItems() {
+ allItems = append(allItems, i)
+ }
+ params.Offset += page.Len()
+ }
+ return allItems, nil
+}
+
+func subtract(setA map[string]bool, setB map[string]bool) map[string]bool {
+ result := make(map[string]bool)
+ for element := range setA {
+ if !setB[element] {
+ result[element] = true
+ }
+ }
+ return result
+}
+
+func jsonReader(rscName string, ob interface{}) io.Reader {
+ j, err := json.Marshal(ob)
+ if err != nil {
+ panic(err)
+ }
+ v := url.Values{}
+ v[rscName] = []string{string(j)}
+ return bytes.NewBufferString(v.Encode())
+}
+
+// GetRemoteGroups fetches all remote groups with their members
+func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remoteGroups map[string]*GroupInfo, groupNameToUUID map[string]string, err error) {
+ remoteGroups = make(map[string]*GroupInfo)
+ groupNameToUUID = make(map[string]string) // Index by group name
+
+ params := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: cfg.ParentGroupUUID,
+ }},
+ }
+ results, err := GetAll(cfg.Client, "groups", params, &GroupList{})
+ if err != nil {
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote groups: %s", err)
+ }
+ for _, item := range results {
+ group := item.(arvados.Group)
+ // Group -> User filter
+ g2uFilter := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: cfg.SysUserUUID,
+ }, {
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "name",
+ Operator: "=",
+ Operand: "can_read",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }, {
+ Attr: "head_kind",
+ Operator: "=",
+ Operand: "arvados#user",
+ }},
+ }
+ // User -> Group filter
+ u2gFilter := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: cfg.SysUserUUID,
+ }, {
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "name",
+ Operator: "=",
+ Operand: "can_write",
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }, {
+ Attr: "tail_kind",
+ Operator: "=",
+ Operand: "arvados#user",
+ }},
+ }
+ g2uLinks, err := GetAll(cfg.Client, "links", g2uFilter, &LinkList{})
+ if err != nil {
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_read) links for group %q: %s", group.Name, err)
+ }
+ u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{})
+ if err != nil {
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_write) links for group %q: %s", group.Name, err)
+ }
+ // Build a list of user ids (email or username) belonging to this group
+ membersSet := make(map[string]bool)
+ u2gLinkSet := make(map[string]bool)
+ for _, l := range u2gLinks {
+ linkedMemberUUID := l.(arvados.Link).TailUUID
+ u2gLinkSet[linkedMemberUUID] = true
+ }
+ for _, item := range g2uLinks {
+ link := item.(arvados.Link)
+ // We may have received an old link pointing to a removed account.
+ if _, found := allUsers[link.HeadUUID]; !found {
+ continue
+ }
+ // The matching User -> Group link may not exist if the link
+ // creation failed on a previous run. If that's the case, don't
+ // include this account on the "previous members" list.
+ if _, found := u2gLinkSet[link.HeadUUID]; !found {
+ continue
+ }
+ memberID, err := GetUserID(allUsers[link.HeadUUID], cfg.UserID)
+ if err != nil {
+ return remoteGroups, groupNameToUUID, err
+ }
+ membersSet[memberID] = true
+ }
+ remoteGroups[group.UUID] = &GroupInfo{
+ Group: group,
+ PreviousMembers: membersSet,
+ CurrentMembers: make(map[string]bool), // Empty set
+ }
+ groupNameToUUID[group.Name] = group.UUID
+ }
+ return remoteGroups, groupNameToUUID, nil
+}
+
+// RemoveMemberFromGroup remove all links related to the membership
+func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
+ if cfg.Verbose {
+ log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Email, user.UUID, group.Name, group.UUID)
+ }
+ var links []interface{}
+ // Search for all group<->user links (both ways)
+ for _, filterset := range [][]arvados.Filter{
+ // Group -> User
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: user.UUID,
+ }},
+ // Group <- User
+ {{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: user.UUID,
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: group.UUID,
+ }},
+ } {
+ l, err := GetAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &LinkList{})
+ if err != nil {
+ userID, _ := GetUserID(user, cfg.UserID)
+ return fmt.Errorf("error getting links needed to remove user %q from group %q: %s", userID, group.Name, err)
+ }
+ for _, link := range l {
+ links = append(links, link)
+ }
+ }
+ for _, item := range links {
+ link := item.(arvados.Link)
+ userID, _ := GetUserID(user, cfg.UserID)
+ if cfg.Verbose {
+ log.Printf("Removing %q permission link for %q on group %q", link.Name, userID, group.Name)
+ }
+ if err := DeleteLink(cfg, link.UUID); err != nil {
+ return fmt.Errorf("error removing user %q from group %q: %s", userID, group.Name, err)
+ }
+ }
+ return nil
+}
+
+// AddMemberToGroup create membership links
+func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
+ var newLink arvados.Link
+ linkData := map[string]string{
+ "owner_uuid": cfg.SysUserUUID,
+ "link_class": "permission",
+ "name": "can_read",
+ "tail_uuid": group.UUID,
+ "head_uuid": user.UUID,
+ }
+ if err := CreateLink(cfg, &newLink, linkData); err != nil {
+ userID, _ := GetUserID(user, cfg.UserID)
+ return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err)
+ }
+ linkData = map[string]string{
+ "owner_uuid": cfg.SysUserUUID,
+ "link_class": "permission",
+ "name": "can_write",
+ "tail_uuid": user.UUID,
+ "head_uuid": group.UUID,
+ }
+ if err := CreateLink(cfg, &newLink, linkData); err != nil {
+ userID, _ := GetUserID(user, cfg.UserID)
+ return fmt.Errorf("error adding user %q -> group %q write permission: %s", userID, group.Name, err)
+ }
+ return nil
+}
+
+// CreateGroup creates a group with groupData parameters, assigns it to dst
+func CreateGroup(cfg *ConfigParams, dst *arvados.Group, groupData map[string]string) error {
+ return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/groups", jsonReader("group", groupData), nil)
+}
+
+// GetGroup fetches a group by its UUID
+func GetGroup(cfg *ConfigParams, dst *arvados.Group, groupUUID string) error {
+ return cfg.Client.RequestAndDecode(&dst, "GET", "/arvados/v1/groups/"+groupUUID, nil, nil)
+}
+
+// CreateLink creates a link with linkData parameters, assigns it to dst
+func CreateLink(cfg *ConfigParams, dst *arvados.Link, linkData map[string]string) error {
+ return cfg.Client.RequestAndDecode(dst, "POST", "/arvados/v1/links", jsonReader("link", linkData), nil)
+}
+
+// DeleteLink deletes a link by its UUID
+func DeleteLink(cfg *ConfigParams, linkUUID string) error {
+ if linkUUID == "" {
+ return fmt.Errorf("cannot delete link with invalid UUID: %q", linkUUID)
+ }
+ return cfg.Client.RequestAndDecode(&arvados.Link{}, "DELETE", "/arvados/v1/links/"+linkUUID, nil, nil)
+}
+
+// GetResourceList fetches res list using params
+func GetResourceList(c *arvados.Client, dst *resourceList, res string, params interface{}) error {
+ return c.RequestAndDecode(dst, "GET", "/arvados/v1/"+res, nil, params)
+}
--- /dev/null
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package main
+
+import (
+ "fmt"
+ "io/ioutil"
+ "os"
+ "strings"
+ "testing"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ . "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ TestingT(t)
+}
+
+type TestSuite struct {
+ cfg *ConfigParams
+ users map[string]arvados.User
+}
+
+func (s *TestSuite) SetUpSuite(c *C) {
+ arvadostest.StartAPI()
+}
+
+func (s *TestSuite) TearDownSuite(c *C) {
+ arvadostest.StopAPI()
+}
+
+func (s *TestSuite) SetUpTest(c *C) {
+ ac := arvados.NewClientFromEnv()
+ u, err := ac.CurrentUser()
+ c.Assert(err, IsNil)
+ // Check that the parent group doesn't exist
+ sysUserUUID := u.UUID[:12] + "000000000000000"
+ gl := arvados.GroupList{}
+ params := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: sysUserUUID,
+ }, {
+ Attr: "name",
+ Operator: "=",
+ Operand: "Externally synchronized groups",
+ }},
+ }
+ ac.RequestAndDecode(&gl, "GET", "/arvados/v1/groups", nil, params)
+ c.Assert(gl.ItemsAvailable, Equals, 0)
+ // Set up config
+ os.Args = []string{"cmd", "somefile.csv"}
+ config, err := GetConfig()
+ c.Assert(err, IsNil)
+ // Confirm that the parent group was created
+ gl = arvados.GroupList{}
+ ac.RequestAndDecode(&gl, "GET", "/arvados/v1/groups", nil, params)
+ c.Assert(gl.ItemsAvailable, Equals, 1)
+ // Config set up complete, save config for further testing
+ s.cfg = &config
+
+ // Fetch current user list
+ ul := arvados.UserList{}
+ params = arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "uuid",
+ Operator: "!=",
+ Operand: s.cfg.SysUserUUID,
+ }},
+ }
+ ac.RequestAndDecode(&ul, "GET", "/arvados/v1/users", nil, params)
+ c.Assert(ul.ItemsAvailable, Not(Equals), 0)
+ s.users = make(map[string]arvados.User)
+ for _, u := range ul.Items {
+ s.users[u.UUID] = u
+ }
+ c.Assert(len(s.users), Not(Equals), 0)
+}
+
+// Clean any membership link and remote group created by the test
+func (s *TestSuite) TearDownTest(c *C) {
+ var dst interface{}
+ // Reset database to fixture state after every test run.
+ err := s.cfg.Client.RequestAndDecode(&dst, "POST", "/database/reset", nil, nil)
+ c.Assert(err, IsNil)
+}
+
+var _ = Suite(&TestSuite{})
+
+// MakeTempCVSFile creates a temp file with data as comma separated values
+func MakeTempCSVFile(data [][]string) (f *os.File, err error) {
+ f, err = ioutil.TempFile("", "test_sync_remote_groups")
+ if err != nil {
+ return
+ }
+ for _, line := range data {
+ fmt.Fprintf(f, "%s\n", strings.Join(line, ","))
+ }
+ err = f.Close()
+ return
+}
+
+// GroupMembershipExists checks that both needed links exist between user and group
+func GroupMembershipExists(ac *arvados.Client, userUUID string, groupUUID string) bool {
+ ll := LinkList{}
+ // Check Group -> User can_read permission
+ params := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: groupUUID,
+ }, {
+ Attr: "name",
+ Operator: "=",
+ Operand: "can_read",
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: userUUID,
+ }},
+ }
+ ac.RequestAndDecode(&ll, "GET", "/arvados/v1/links", nil, params)
+ if ll.Len() != 1 {
+ return false
+ }
+ // Check User -> Group can_write permission
+ params = arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "link_class",
+ Operator: "=",
+ Operand: "permission",
+ }, {
+ Attr: "head_uuid",
+ Operator: "=",
+ Operand: groupUUID,
+ }, {
+ Attr: "name",
+ Operator: "=",
+ Operand: "can_write",
+ }, {
+ Attr: "tail_uuid",
+ Operator: "=",
+ Operand: userUUID,
+ }},
+ }
+ ac.RequestAndDecode(&ll, "GET", "/arvados/v1/links", nil, params)
+ if ll.Len() != 1 {
+ return false
+ }
+ return true
+}
+
+// If named group exists, return its UUID
+func RemoteGroupExists(cfg *ConfigParams, groupName string) (uuid string, err error) {
+ gl := arvados.GroupList{}
+ params := arvados.ResourceListParams{
+ Filters: []arvados.Filter{{
+ Attr: "name",
+ Operator: "=",
+ Operand: groupName,
+ }, {
+ Attr: "owner_uuid",
+ Operator: "=",
+ Operand: cfg.ParentGroupUUID,
+ }, {
+ Attr: "group_class",
+ Operator: "=",
+ Operand: "role",
+ }},
+ }
+ err = cfg.Client.RequestAndDecode(&gl, "GET", "/arvados/v1/groups", nil, params)
+ if err != nil {
+ return "", err
+ }
+ if gl.ItemsAvailable == 0 {
+ // No group with this name
+ uuid = ""
+ } else if gl.ItemsAvailable == 1 {
+ // Group found
+ uuid = gl.Items[0].UUID
+ } else {
+ // This should never happen
+ uuid = ""
+ err = fmt.Errorf("more than 1 group found with the same name and parent")
+ }
+ return
+}
+
+func (s *TestSuite) TestParseFlagsWithPositionalArgument(c *C) {
+ cfg := ConfigParams{}
+ os.Args = []string{"cmd", "-verbose", "/tmp/somefile.csv"}
+ err := ParseFlags(&cfg)
+ c.Assert(err, IsNil)
+ c.Check(cfg.Path, Equals, "/tmp/somefile.csv")
+ c.Check(cfg.Verbose, Equals, true)
+}
+
+func (s *TestSuite) TestParseFlagsWithoutPositionalArgument(c *C) {
+ os.Args = []string{"cmd", "-verbose"}
+ err := ParseFlags(&ConfigParams{})
+ c.Assert(err, NotNil)
+}
+
+func (s *TestSuite) TestGetUserID(c *C) {
+ u := arvados.User{
+ Email: "testuser@example.com",
+ Username: "Testuser",
+ }
+ email, err := GetUserID(u, "email")
+ c.Assert(err, IsNil)
+ c.Check(email, Equals, "testuser@example.com")
+ _, err = GetUserID(u, "bogus")
+ c.Assert(err, NotNil)
+}
+
+func (s *TestSuite) TestGetConfig(c *C) {
+ os.Args = []string{"cmd", "/tmp/somefile.csv"}
+ cfg, err := GetConfig()
+ c.Assert(err, IsNil)
+ c.Check(cfg.SysUserUUID, NotNil)
+ c.Check(cfg.Client, NotNil)
+ c.Check(cfg.ParentGroupUUID, NotNil)
+ c.Check(cfg.ParentGroupName, Equals, "Externally synchronized groups")
+}
+
+// Ignore leading & trailing spaces on group & users names
+func (s *TestSuite) TestIgnoreSpaces(c *C) {
+ activeUserEmail := s.users[arvadostest.ActiveUserUUID].Email
+ activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ // Confirm that the groups don't exist
+ for _, groupName := range []string{"TestGroup1", "TestGroup2", "Test Group 3"} {
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ }
+ data := [][]string{
+ {" TestGroup1", activeUserEmail},
+ {"TestGroup2 ", " " + activeUserEmail},
+ {" Test Group 3 ", activeUserEmail + " "},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Check that 3 groups were created correctly, and have the active user as
+ // a member.
+ for _, groupName := range []string{"TestGroup1", "TestGroup2", "Test Group 3"} {
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ }
+}
+
+// The absence of a user membership on the CSV file implies its removal
+func (s *TestSuite) TestMembershipRemoval(c *C) {
+ activeUserEmail := s.users[arvadostest.ActiveUserUUID].Email
+ activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ data := [][]string{
+ {"TestGroup1", activeUserEmail},
+ {"TestGroup2", activeUserEmail},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Confirm that memberships exist
+ for _, groupName := range []string{"TestGroup1", "TestGroup2"} {
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ }
+ // New CSV with one previous membership missing
+ data = [][]string{
+ {"TestGroup1", activeUserEmail},
+ }
+ tmpfile2, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile2.Name()) // clean up
+ s.cfg.Path = tmpfile2.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Confirm TestGroup1 membership still exist
+ groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup1")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+ // Confirm TestGroup2 membership was removed
+ groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup2")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, false)
+}
+
+// If a group doesn't exist on the system, create it before adding users
+func (s *TestSuite) TestAutoCreateGroupWhenNotExisting(c *C) {
+ groupName := "Testers"
+ // Confirm that group doesn't exist
+ groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ // Make a tmp CSV file
+ data := [][]string{
+ {groupName, s.users[arvadostest.ActiveUserUUID].Email},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // "Testers" group should now exist
+ groupUUID, err = RemoteGroupExists(s.cfg, groupName)
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ // active user should be a member
+ c.Assert(GroupMembershipExists(s.cfg.Client, arvadostest.ActiveUserUUID, groupUUID), Equals, true)
+}
+
+// Users listed on the file that don't exist on the system are ignored
+func (s *TestSuite) TestIgnoreNonexistantUsers(c *C) {
+ activeUserEmail := s.users[arvadostest.ActiveUserUUID].Email
+ activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ // Confirm that group doesn't exist
+ groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup4")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ // Create file & run command
+ data := [][]string{
+ {"TestGroup4", "nonexistantuser@unknowndomain.com"}, // Processed first
+ {"TestGroup4", activeUserEmail},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Confirm that memberships exist
+ groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup4")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+}
+
+// Users listed on the file that don't exist on the system are ignored
+func (s *TestSuite) TestIgnoreEmptyFields(c *C) {
+ activeUserEmail := s.users[arvadostest.ActiveUserUUID].Email
+ activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ // Confirm that group doesn't exist
+ groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup4")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ // Create file & run command
+ data := [][]string{
+ {"", activeUserEmail}, // Empty field
+ {"TestGroup5", ""}, // Empty field
+ {"TestGroup4", activeUserEmail},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ err = doMain(s.cfg)
+ c.Assert(err, IsNil)
+ // Confirm that memberships exist
+ groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup4")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+}
+
+// Instead of emails, use username as identifier
+func (s *TestSuite) TestUseUsernames(c *C) {
+ activeUserName := s.users[arvadostest.ActiveUserUUID].Username
+ activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
+ // Confirm that group doesn't exist
+ groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup1")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Equals, "")
+ // Create file & run command
+ data := [][]string{
+ {"TestGroup1", activeUserName},
+ }
+ tmpfile, err := MakeTempCSVFile(data)
+ c.Assert(err, IsNil)
+ defer os.Remove(tmpfile.Name()) // clean up
+ s.cfg.Path = tmpfile.Name()
+ s.cfg.UserID = "username"
+ err = doMain(s.cfg)
+ s.cfg.UserID = "email"
+ c.Assert(err, IsNil)
+ // Confirm that memberships exist
+ groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup1")
+ c.Assert(err, IsNil)
+ c.Assert(groupUUID, Not(Equals), "")
+ c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+}
projects / arvados.git / commitdiff