21126: Merge branch 'main' into 21126-trash-when-ro

[arvados.git] / sdk / python / arvados / commands / federation_migrate.py

diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index 5c1bb29e764c549d1612bb13f2890a076866fc74..770e1609db6ec60dc39567678986c53f3a6f5a35 100755 (executable)
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -24,6 +24,7 @@ import os
 import hashlib
 import re
 from arvados._version import __version__
+from . import _util as arv_cmd
 
 EMAIL=0
 USERNAME=1
@@ -43,10 +44,10 @@ def connect_clusters(args):
                 host = r[0]
                 token = r[1]
                 print("Contacting %s" % (host))
-                arv = arvados.api(host=host, token=token, cache=False)
+                arv = arvados.api(host=host, token=token, cache=False, num_retries=args.retries)
                 clusters[arv._rootDesc["uuidPrefix"]] = arv
     else:
-        arv = arvados.api(cache=False)
+        arv = arvados.api(cache=False, num_retries=args.retries)
         rh = arv._rootDesc["remoteHosts"]
         tok = arv.api_client_authorizations().current().execute()
         token = "v2/%s/%s" % (tok["uuid"], tok["api_token"])
@@ -96,13 +97,12 @@ def fetch_users(clusters, loginCluster):
     by_email = {}
     by_username = {}
 
-    users = []
-    for c, arv in clusters.items():
-        print("Getting user list from %s" % c)
-        ul = arvados.util.list_all(arv.users().list, bypass_federation=True)
-        for l in ul:
-            if l["uuid"].startswith(c):
-                users.append(l)
+    users = [
+        user
+        for prefix, arv in clusters.items()
+        for user in arvados.util.keyset_list_all(arv.users().list, bypass_federation=True)
+        if user['uuid'].startswith(prefix)
+    ]
 
     # Users list is sorted by email
     # Go through users and collect users with same email
@@ -110,7 +110,7 @@ def fetch_users(clusters, loginCluster):
     # call add_accum_rows() to generate the report rows with
     # the "home cluster" set, and also fill in the by_email table.
 
-    users = sorted(users, key=lambda u: u["email"]+"::"+(u["username"] or "")+"::"+u["uuid"])
+    users.sort(key=lambda u: (u["email"], u["username"] or "", u["uuid"]))
 
     accum = []
     lastemail = None
@@ -326,7 +326,10 @@ def migrate_user(args, migratearv, email, new_user_uuid, old_user_uuid):
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Migrate users to federated identity, see https://doc.arvados.org/admin/merge-remote-account.html')
+    parser = argparse.ArgumentParser(
+        description='Migrate users to federated identity, see https://doc.arvados.org/admin/merge-remote-account.html',
+        parents=[arv_cmd.retry_opt],
+    )
     parser.add_argument(
         '--version', action='version', version="%s %s" % (sys.argv[0], __version__),
         help='Print version and exit.')