1 // Copyright (C) The Arvados Authors. All rights reserved.
3 // SPDX-License-Identifier: AGPL-3.0
21 "git.arvados.org/arvados.git/lib/cmd"
22 "git.arvados.org/arvados.git/sdk/go/arvados"
27 type resourceList interface {
29 GetItems() []interface{}
32 // UserList implements resourceList interface
33 type UserList struct {
37 // Len returns the amount of items this list holds
38 func (l UserList) Len() int {
42 // GetItems returns the list of items
43 func (l UserList) GetItems() (out []interface{}) {
44 for _, item := range l.Items {
45 out = append(out, item)
51 cfg, err := GetConfig()
56 if err := doMain(&cfg); err != nil {
61 type ConfigParams struct {
63 Client *arvados.Client
65 CurrentUser arvados.User
66 DeactivateUnlisted bool
74 func ParseFlags(cfg *ConfigParams) error {
75 // Acceptable attributes to identify a user on the CSV file
76 userIDOpts := map[string]bool{
77 "email": true, // default
81 flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
82 flags.Usage = func() {
83 usageStr := `Synchronize remote users into Arvados from a CSV format file with 5 columns:
84 * 1st: User Identifier (email or username)
87 * 4th: Active status (0 or 1)
88 * 5th: Admin status (0 or 1)`
89 fmt.Fprintf(flags.Output(), "%s\n\n", usageStr)
90 fmt.Fprintf(flags.Output(), "Usage:\n%s [OPTIONS] <input-file.csv>\n\n", os.Args[0])
91 fmt.Fprintf(flags.Output(), "Options:\n")
95 caseInsensitive := flags.Bool(
98 "Performs case insensitive matching on user IDs. Always ON when using 'email' user IDs.")
99 deactivateUnlisted := flags.Bool(
100 "deactivate-unlisted",
102 "Deactivate users that are not in the input file.")
103 userID := flags.String(
106 "Attribute by which every user is identified. Valid values are: email and username.")
107 verbose := flags.Bool(
110 "Log informational messages.")
111 getVersion := flags.Bool(
114 "Print version information and exit.")
116 if ok, code := cmd.ParseFlags(flags, os.Args[0], os.Args[1:], "input-file.csv", os.Stderr); !ok {
118 } else if *getVersion {
119 fmt.Printf("%s %s\n", os.Args[0], version)
123 // Input file as a required positional argument
124 if flags.NArg() == 0 {
125 return fmt.Errorf("please provide a path to an input file")
126 } else if flags.NArg() > 1 {
127 return fmt.Errorf("please provide just one input file argument")
129 srcPath := &os.Args[len(os.Args)-1]
133 return fmt.Errorf("input file path invalid")
135 if !userIDOpts[*userID] {
137 for opt := range userIDOpts {
138 options = append(options, opt)
140 return fmt.Errorf("user ID must be one of: %s", strings.Join(options, ", "))
142 if *userID == "email" {
143 // Always do case-insensitive email addresses matching
144 *caseInsensitive = true
147 cfg.CaseInsensitive = *caseInsensitive
148 cfg.DeactivateUnlisted = *deactivateUnlisted
151 cfg.Verbose = *verbose
156 // GetConfig sets up a ConfigParams struct
157 func GetConfig() (cfg ConfigParams, err error) {
158 err = ParseFlags(&cfg)
163 cfg.Client = arvados.NewClientFromEnv()
165 // Check current user permissions
166 u, err := cfg.Client.CurrentUser()
168 return cfg, fmt.Errorf("error getting the current user: %s", err)
171 return cfg, fmt.Errorf("current user %q is not an admin user", u.UUID)
174 log.Printf("Running as admin user %q (%s)", u.Email, u.UUID)
184 err = cfg.Client.RequestAndDecode(&ac, "GET", "arvados/v1/config", nil, nil)
186 return cfg, fmt.Errorf("error getting the exported config: %s", err)
188 if ac.Login.LoginCluster != "" && ac.Login.LoginCluster != ac.ClusterID {
189 return cfg, fmt.Errorf("cannot run on a cluster other than the login cluster")
191 cfg.SysUserUUID = ac.ClusterID + "-tpzed-000000000000000"
192 cfg.AnonUserUUID = ac.ClusterID + "-tpzed-anonymouspublic"
193 cfg.ClusterID = ac.ClusterID
198 // GetUserID returns the correct user id value depending on the selector
199 func GetUserID(u arvados.User, idSelector string) (string, error) {
204 return u.Username, nil
206 return "", fmt.Errorf("cannot identify user by %q selector", idSelector)
210 func doMain(cfg *ConfigParams) error {
211 // Try opening the input file early, just in case there's a problem.
212 f, err := os.Open(cfg.Path)
214 return fmt.Errorf("error opening input file: %s", err)
219 if cfg.UserID == "username" && cfg.CaseInsensitive {
220 iCaseLog = " - username matching requested to be case-insensitive"
222 log.Printf("%s %s started. Using %q as users id%s", os.Args[0], version, cfg.UserID, iCaseLog)
224 allUsers := make(map[string]arvados.User)
225 userIDToUUID := make(map[string]string) // Index by email or username
226 dupedEmails := make(map[string][]arvados.User)
227 emptyUserIDs := []string{}
228 processedUsers := make(map[string]bool)
229 results, err := GetAll(cfg.Client, "users", arvados.ResourceListParams{}, &UserList{})
231 return fmt.Errorf("error getting all users: %s", err)
233 log.Printf("Found %d users in cluster %q", len(results), cfg.ClusterID)
234 localUserUuidRegex := regexp.MustCompile(fmt.Sprintf("^%s-tpzed-[0-9a-z]{15}$", cfg.ClusterID))
235 for _, item := range results {
236 u := item.(arvados.User)
239 if !localUserUuidRegex.MatchString(u.UUID) {
241 log.Printf("Remote user %q (%s) won't be considered for processing", u.Email, u.UUID)
246 // Duplicated user id check
247 uID, err := GetUserID(u, cfg.UserID)
252 emptyUserIDs = append(emptyUserIDs, u.UUID)
253 log.Printf("Empty %s found in user %s - ignoring", cfg.UserID, u.UUID)
256 if cfg.CaseInsensitive {
257 uID = strings.ToLower(uID)
259 if alreadySeenUUID, found := userIDToUUID[uID]; found {
260 if cfg.UserID == "username" && uID != "" {
261 return fmt.Errorf("case insensitive collision for username %q between %q and %q", uID, u.UUID, alreadySeenUUID)
262 } else if cfg.UserID == "email" && uID != "" {
263 log.Printf("Duplicated email %q found in user %s - ignoring", uID, u.UUID)
264 if len(dupedEmails[uID]) == 0 {
265 dupedEmails[uID] = []arvados.User{allUsers[alreadySeenUUID]}
267 dupedEmails[uID] = append(dupedEmails[uID], u)
268 delete(allUsers, alreadySeenUUID) // Skip even the first occurrence,
269 // for security purposes.
274 log.Printf("Seen user %q (%s)", uID, u.UUID)
276 userIDToUUID[uID] = u.UUID
278 processedUsers[u.UUID] = false
281 loadedRecords, err := LoadInputFile(f)
283 return fmt.Errorf("reading input file %q: %s", cfg.Path, err)
285 log.Printf("Loaded %d records from input file", len(loadedRecords))
287 updatesSucceeded := map[string]bool{}
288 updatesFailed := map[string]bool{}
289 updatesSkipped := map[string]bool{}
291 for _, record := range loadedRecords {
292 if cfg.CaseInsensitive {
293 record.UserID = strings.ToLower(record.UserID)
295 recordUUID := userIDToUUID[record.UserID]
296 processedUsers[recordUUID] = true
297 if cfg.UserID == "email" && record.UserID == cfg.CurrentUser.Email {
298 updatesSkipped[recordUUID] = true
299 log.Printf("Skipping current user %q (%s) from processing", record.UserID, cfg.CurrentUser.UUID)
302 if updated, err := ProcessRecord(cfg, record, userIDToUUID, allUsers); err != nil {
303 log.Printf("error processing record %q: %s", record.UserID, err)
304 updatesFailed[recordUUID] = true
306 updatesSucceeded[recordUUID] = true
310 if cfg.DeactivateUnlisted {
311 for userUUID, user := range allUsers {
312 if shouldSkip(cfg, user) {
313 updatesSkipped[userUUID] = true
314 log.Printf("Skipping unlisted user %q (%s) from deactivating", user.Email, user.UUID)
317 if !processedUsers[userUUID] && allUsers[userUUID].IsActive {
319 log.Printf("Deactivating unlisted user %q (%s)", user.Username, user.UUID)
321 var updatedUser arvados.User
322 if err := UnsetupUser(cfg.Client, user.UUID, &updatedUser); err != nil {
323 log.Printf("error deactivating unlisted user %q: %s", user.UUID, err)
324 updatesFailed[userUUID] = true
326 allUsers[userUUID] = updatedUser
327 updatesSucceeded[userUUID] = true
333 log.Printf("User update successes: %d, skips: %d, failures: %d", len(updatesSucceeded), len(updatesSkipped), len(updatesFailed))
336 if len(dupedEmails) > 0 {
337 emails := make([]string, len(dupedEmails))
339 for e := range dupedEmails {
343 errors = append(errors, fmt.Sprintf("skipped %d duplicated email address(es) in the cluster's local user list: %v", len(dupedEmails), emails))
345 if len(emptyUserIDs) > 0 {
346 errors = append(errors, fmt.Sprintf("skipped %d user account(s) with empty %s: %v", len(emptyUserIDs), cfg.UserID, emptyUserIDs))
349 return fmt.Errorf("%s", strings.Join(errors, "\n"))
355 func shouldSkip(cfg *ConfigParams, user arvados.User) bool {
357 case cfg.SysUserUUID, cfg.AnonUserUUID:
359 case cfg.CurrentUser.UUID:
365 type userRecord struct {
373 func needsUpdating(user arvados.User, record userRecord) bool {
374 userData := userRecord{"", user.FirstName, user.LastName, user.IsActive, user.IsAdmin}
375 recordData := userRecord{"", record.FirstName, record.LastName, record.Active, record.Active && record.Admin}
376 return userData != recordData
379 // ProcessRecord creates or updates a user based on the given record
380 func ProcessRecord(cfg *ConfigParams, record userRecord, userIDToUUID map[string]string, allUsers map[string]arvados.User) (bool, error) {
382 log.Printf("Processing record for user %q", record.UserID)
385 wantedActiveStatus := strconv.FormatBool(record.Active)
386 wantedAdminStatus := strconv.FormatBool(record.Active && record.Admin)
387 createRequired := false
388 updateRequired := false
389 // Check if user exists, set its active & admin status.
390 var user arvados.User
391 recordUUID := userIDToUUID[record.UserID]
392 user, found := allUsers[recordUUID]
395 log.Printf("User %q does not exist, creating", record.UserID)
397 createRequired = true
398 err := CreateUser(cfg.Client, &user, map[string]string{
399 cfg.UserID: record.UserID,
400 "first_name": record.FirstName,
401 "last_name": record.LastName,
402 "is_active": wantedActiveStatus,
403 "is_admin": wantedAdminStatus,
406 return false, fmt.Errorf("error creating user %q: %s", record.UserID, err)
408 } else if needsUpdating(user, record) {
409 updateRequired = true
411 if !user.IsActive && cfg.Verbose {
412 log.Printf("User %q (%s) is inactive, activating", record.UserID, user.UUID)
414 // Here we assume the 'setup' is done elsewhere if needed.
415 err := UpdateUser(cfg.Client, user.UUID, &user, map[string]string{
416 "first_name": record.FirstName,
417 "last_name": record.LastName,
418 "is_active": wantedActiveStatus,
419 "is_admin": wantedAdminStatus,
422 return false, fmt.Errorf("error updating user %q: %s", record.UserID, err)
425 fnChanged := user.FirstName != record.FirstName
426 lnChanged := user.LastName != record.LastName
427 if fnChanged || lnChanged {
428 err := UpdateUser(cfg.Client, user.UUID, &user, map[string]string{
429 "first_name": record.FirstName,
430 "last_name": record.LastName,
433 return false, fmt.Errorf("error updating user %q: %s", record.UserID, err)
438 log.Printf("User %q is active, deactivating", record.UserID)
440 err := UnsetupUser(cfg.Client, user.UUID, &user)
442 return false, fmt.Errorf("error deactivating user %q: %s", record.UserID, err)
448 log.Printf("Created user %q", record.UserID)
451 log.Printf("Updated user %q", record.UserID)
454 return createRequired || updateRequired, nil
457 // LoadInputFile reads the input file and returns a list of user records
458 func LoadInputFile(f *os.File) (loadedRecords []userRecord, err error) {
460 csvReader := csv.NewReader(f)
461 loadedRecords = make([]userRecord, 0)
464 record, e := csvReader.Read()
470 err = fmt.Errorf("parsing error at line %d: %s", lineNo, e)
473 if len(record) != 5 {
474 err = fmt.Errorf("parsing error at line %d: expected 5 fields, found %d", lineNo, len(record))
477 userID := strings.ToLower(strings.TrimSpace(record[0]))
478 firstName := strings.TrimSpace(record[1])
479 lastName := strings.TrimSpace(record[2])
480 active := strings.TrimSpace(record[3])
481 admin := strings.TrimSpace(record[4])
482 if userID == "" || firstName == "" || lastName == "" || active == "" || admin == "" {
483 err = fmt.Errorf("parsing error at line %d: fields cannot be empty", lineNo)
486 activeBool, err := strconv.ParseBool(active)
488 return nil, fmt.Errorf("parsing error at line %d: active status not recognized", lineNo)
490 adminBool, err := strconv.ParseBool(admin)
492 return nil, fmt.Errorf("parsing error at line %d: admin status not recognized", lineNo)
494 loadedRecords = append(loadedRecords, userRecord{
496 FirstName: firstName,
502 return loadedRecords, nil
505 // GetAll adds all objects of type 'resource' to the 'allItems' list
506 func GetAll(c *arvados.Client, res string, params arvados.ResourceListParams, page resourceList) (allItems []interface{}, err error) {
507 // Use the maximum page size the server allows
509 params.Limit = &limit
511 params.Order = "uuid"
513 if err = GetResourceList(c, &page, res, params); err != nil {
516 // Have we finished paging?
520 allItems = append(allItems, page.GetItems()...)
521 params.Offset += page.Len()
526 func jsonReader(rscName string, ob interface{}) io.Reader {
527 j, err := json.Marshal(ob)
532 v[rscName] = []string{string(j)}
533 return bytes.NewBufferString(v.Encode())
536 // GetResourceList fetches res list using params
537 func GetResourceList(c *arvados.Client, dst *resourceList, res string, params interface{}) error {
538 return c.RequestAndDecode(dst, "GET", "/arvados/v1/"+res, nil, params)
541 // CreateUser creates a user with userData parameters, assigns it to dst
542 func CreateUser(c *arvados.Client, dst *arvados.User, userData map[string]string) error {
543 return c.RequestAndDecode(dst, "POST", "/arvados/v1/users", jsonReader("user", userData), nil)
546 // UpdateUser updates a user with userData parameters
547 func UpdateUser(c *arvados.Client, userUUID string, dst *arvados.User, userData map[string]string) error {
548 return c.RequestAndDecode(&dst, "PUT", "/arvados/v1/users/"+userUUID, jsonReader("user", userData), nil)
551 // UnsetupUser deactivates a user
552 func UnsetupUser(c *arvados.Client, userUUID string, dst *arvados.User) error {
553 return c.RequestAndDecode(&dst, "POST", "/arvados/v1/users/"+userUUID+"/unsetup", nil, nil)