arvados_cluster_ids:
type: string[]
outputSource: start/cluster_id
+ superuser_tokens:
+ type: string[]
+ outputSource: start/superuser_token
acr:
type: string?
outputSource: in_acr
arvado_api_host_insecure:
type: boolean
outputSource: insecure
+ arvbox_containers:
+ type: string[]
+ outputSource: containers
steps:
mkdir:
in:
outputs:
test_user_uuid: string
test_user_token: string
-arguments: [python2, $(inputs.make_user_script)]
\ No newline at end of file
+arguments: [python, $(inputs.make_user_script)]
import argparse
import hmac
import urllib.parse
+import os
def main():
parser = argparse.ArgumentParser(description='Migrate users to federated identity, see https://doc.arvados.org/admin/merge-remote-account.html')
- parser.add_argument('--tokens', type=str, required=True)
+ parser.add_argument('--tokens', type=str, required=False)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--report', type=str, help="Generate report .csv file listing users by email address and their associated Arvados accounts")
group.add_argument('--migrate', type=str, help="Consume report .csv and migrate users to designated Arvados accounts")
+ group.add_argument('--dry-run', type=str, help="Consume report .csv and report how user would be migrated to designated Arvados accounts")
group.add_argument('--check', action="store_true", help="Check that tokens are usable and the federation is well connected")
args = parser.parse_args()
clusters = {}
errors = []
- print("Reading %s" % args.tokens)
- with open(args.tokens, "rt") as f:
- for r in csv.reader(f):
- host = r[0]
- token = r[1]
- print("Contacting %s" % (host))
- arv = arvados.api(host=host, token=token, cache=False)
- try:
- cur = arv.users().current().execute()
- arv.api_client_authorizations().list(limit=1).execute()
- except arvados.errors.ApiError as e:
- errors.append("checking token for %s: %s" % (host, e))
- errors.append(' This script requires a token issued to a trusted client in order to manipulate access tokens.')
- errors.append(' See "Trusted client setting" in https://doc.arvados.org/install/install-workbench-app.html')
- errors.append(' and https://doc.arvados.org/api/tokens.html')
- continue
+ loginCluster = None
+ if args.tokens:
+ print("Reading %s" % args.tokens)
+ with open(args.tokens, "rt") as f:
+ for r in csv.reader(f):
+ host = r[0]
+ token = r[1]
+ print("Contacting %s" % (host))
+ arv = arvados.api(host=host, token=token, cache=False)
+ clusters[arv._rootDesc["uuidPrefix"]] = arv
+ else:
+ arv = arvados.api(cache=False)
+ rh = arv._rootDesc["remoteHosts"]
+ tok = arv.api_client_authorizations().current().execute()
+ token = "v2/%s/%s" % (tok["uuid"], tok["api_token"])
- if not cur["is_admin"]:
- errors.append("Not admin of %s" % host)
- continue
+ for k,v in rh.items():
+ arv = arvados.api(host=v, token=token, cache=False, insecure=os.environ.get("ARVADOS_API_HOST_INSECURE"))
+ config = arv.configs().get().execute()
+ if config["Login"]["LoginCluster"] != "" and loginCluster is None:
+ loginCluster = config["Login"]["LoginCluster"]
+ clusters[k] = arv
- clusters[arv._rootDesc["uuidPrefix"]] = arv
+ print("Checking that the federation is well connected")
+ for arv in clusters.values():
+ config = arv.configs().get().execute()
+ if loginCluster and config["Login"]["LoginCluster"] != loginCluster and config["ClusterID"] != loginCluster:
+ errors.append("Inconsistent login cluster configuration, expected '%s' on %s but was '%s'" % (loginCluster, config["ClusterID"], config["Login"]["LoginCluster"]))
+ continue
+ try:
+ cur = arv.users().current().execute()
+ #arv.api_client_authorizations().list(limit=1).execute()
+ except arvados.errors.ApiError as e:
+ errors.append("checking token for %s %s" % (arv._rootDesc["rootUrl"], e))
+ errors.append(' This script requires a token issued to a trusted client in order to manipulate access tokens.')
+ errors.append(' See "Trusted client setting" in https://doc.arvados.org/install/install-workbench-app.html')
+ errors.append(' and https://doc.arvados.org/api/tokens.html')
+ continue
+ if not cur["is_admin"]:
+ errors.append("Not admin of %s" % host)
+ continue
- print("Checking that the federation is well connected")
- for v in clusters.values():
for r in clusters:
- if r != v._rootDesc["uuidPrefix"] and r not in v._rootDesc["remoteHosts"]:
- errors.append("%s is missing from remoteHosts of %s" % (r, v._rootDesc["uuidPrefix"]))
- for r in v._rootDesc["remoteHosts"]:
+ if r != arv._rootDesc["uuidPrefix"] and r not in arv._rootDesc["remoteHosts"]:
+ errors.append("%s is missing from remoteHosts of %s" % (r, arv._rootDesc["uuidPrefix"]))
+ for r in arv._rootDesc["remoteHosts"]:
if r != "*" and r not in clusters:
- print("WARNING: %s is federated with %s but %s is missing from the tokens file or the token is invalid" % (v._rootDesc["uuidPrefix"], r, r))
+ print("WARNING: %s is federated with %s but %s is missing from the tokens file or the token is invalid" % (arv._rootDesc["uuidPrefix"], r, r))
if errors:
for e in errors:
print("Tokens file passed checks")
exit(0)
- if args.report:
- users = []
- for c, arv in clusters.items():
- print("Getting user list from %s" % c)
- ul = arvados.util.list_all(arv.users().list)
- for l in ul:
- if l["uuid"].startswith(c):
- users.append(l)
+ rows = []
+ by_email = {}
- out = csv.writer(open(args.report, "wt"))
+ users = []
+ for c, arv in clusters.items():
+ print("Getting user list from %s" % c)
+ ul = arvados.util.list_all(arv.users().list)
+ for l in ul:
+ if l["uuid"].startswith(c):
+ users.append(l)
- out.writerow(("email", "user uuid", "primary cluster/user"))
+ users = sorted(users, key=lambda u: u["email"]+"::"+(u["username"] or "")+"::"+u["uuid"])
- users = sorted(users, key=lambda u: u["email"]+"::"+u["uuid"])
+ accum = []
+ lastemail = None
+ for u in users:
+ if u["uuid"].endswith("-anonymouspublic") or u["uuid"].endswith("-000000000000000"):
+ continue
+ if lastemail == None:
+ lastemail = u["email"]
+ if u["email"] == lastemail:
+ accum.append(u)
+ else:
+ homeuuid = None
+ for a in accum:
+ if homeuuid is None:
+ homeuuid = a["uuid"]
+ if a["uuid"] != homeuuid:
+ homeuuid = ""
+ for a in accum:
+ r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
+ by_email.setdefault(a["email"], {})
+ by_email[a["email"]][a["uuid"]] = r
+ rows.append(r)
+ lastemail = u["email"]
+ accum = [u]
- accum = []
- lastemail = None
- for u in users:
- if u["uuid"].endswith("-anonymouspublic") or u["uuid"].endswith("-000000000000000"):
- continue
- if lastemail == None:
- lastemail = u["email"]
- if u["email"] == lastemail:
- accum.append(u)
- else:
- homeuuid = None
- for a in accum:
- if homeuuid is None:
- homeuuid = a["uuid"]
- if a["uuid"] != homeuuid:
- homeuuid = ""
- for a in accum:
- out.writerow((a["email"], a["uuid"], homeuuid[0:5]))
- lastemail = u["email"]
- accum = [u]
-
- homeuuid = None
- for a in accum:
- if homeuuid is None:
- homeuuid = a["uuid"]
- if a["uuid"] != homeuuid:
- homeuuid = ""
- for a in accum:
- out.writerow((a["email"], a["uuid"], homeuuid[0:5]))
+ homeuuid = None
+ for a in accum:
+ if homeuuid is None:
+ homeuuid = a["uuid"]
+ if a["uuid"] != homeuuid:
+ homeuuid = ""
+ for a in accum:
+ r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
+ by_email.setdefault(a["email"], {})
+ by_email[a["email"]][a["uuid"]] = r
+ rows.append(r)
+ if args.report:
+ out = csv.writer(open(args.report, "wt"))
+ out.writerow(("email", "username", "user uuid", "home cluster"))
+ for r in rows:
+ out.writerow(r)
print("Wrote %s" % args.report)
+ return
+
+ if args.migrate or args.dry_run:
+ if args.dry_run:
+ print("Performing dry run")
- if args.migrate:
rows = []
- by_email = {}
- with open(args.migrate, "rt") as f:
+
+ with open(args.migrate or args.dry_run, "rt") as f:
for r in csv.reader(f):
if r[0] == "email":
continue
- by_email.setdefault(r[0], [])
- by_email[r[0]].append(r)
+ by_email.setdefault(r[0], {})
+ by_email[r[0]][r[2]] = r
rows.append(r)
+
for r in rows:
email = r[0]
- old_user_uuid = r[1]
- userhome = r[2]
+ username = r[1]
+ old_user_uuid = r[2]
+ userhome = r[3]
if userhome == "":
print("(%s) Skipping %s, no home cluster specified" % (email, old_user_uuid))
if old_user_uuid.startswith(userhome):
+ migratecluster = old_user_uuid[0:5]
+ migratearv = clusters[migratecluster]
+ if migratearv.users().get(uuid=old_user_uuid).execute()["username"] != username:
+ print("(%s) Updating username of %s to '%s' on %s" % (email, old_user_uuid, username, migratecluster))
+ if not args.dry_run:
+ try:
+ conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
+ migratearv.users().update(uuid=old_user_uuid, body={"user": {"username": username}}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, old_user_uuid, username, migratecluster, e))
continue
candidates = []
- for b in by_email[email]:
- if b[1].startswith(userhome):
+ conflict = False
+ for b in by_email[email].values():
+ if b[2].startswith(userhome):
candidates.append(b)
+ if b[1] != username and b[3] == userhome:
+ print("(%s) Cannot migrate %s, conflicting usernames %s and %s" % (email, old_user_uuid, b[1], username))
+ conflict = True
+ break
+ if conflict:
+ continue
if len(candidates) == 0:
if len(userhome) == 5 and userhome not in clusters:
print("(%s) Cannot migrate %s, unknown home cluster %s (typo?)" % (email, old_user_uuid, userhome))
+ continue
+ print("(%s) No user listed with same email to migrate %s to %s, will create new user with username '%s'" % (email, old_user_uuid, userhome, username))
+ if not args.dry_run:
+ newhomecluster = userhome[0:5]
+ homearv = clusters[userhome]
+ user = None
+ try:
+ conflicts = homearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ homearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
+ user = homearv.users().create(body={"user": {"email": email, "username": username}}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not create user: %s" % (email, str(e)))
+ continue
+
+ tup = (email, username, user["uuid"], userhome)
else:
- print("(%s) No user listed with same email to migrate %s to %s" % (email, old_user_uuid, userhome))
- continue
+ # dry run
+ tup = (email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome)
+ by_email[email][tup[2]] = tup
+ candidates.append(tup)
if len(candidates) > 1:
print("(%s) Multiple users listed to migrate %s to %s, use full uuid" % (email, old_user_uuid, userhome))
continue
- new_user_uuid = candidates[0][1]
+ new_user_uuid = candidates[0][2]
# cluster where the migration is happening
- migratecluster = old_user_uuid[0:5]
- migratearv = clusters[migratecluster]
-
- # the user's new home cluster
- newhomecluster = userhome[0:5]
- homearv = clusters[newhomecluster]
-
- # create a token for the new user and salt it for the
- # migration cluster, then use it to access the migration
- # cluster as the new user once before merging to ensure
- # the new user is known on that cluster.
- try:
- newtok = homearv.api_client_authorizations().create(body={
- "api_client_authorization": {'owner_uuid': new_user_uuid}}).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Could not create API token for %s: %s" % (email, new_user_uuid, e))
- continue
+ for arv in clusters.values():
+ migratecluster = arv._rootDesc["uuidPrefix"]
+ migratearv = clusters[migratecluster]
- salted = 'v2/' + newtok["uuid"] + '/' + hmac.new(newtok["api_token"].encode(),
- msg=migratecluster.encode(),
- digestmod='sha1').hexdigest()
- try:
- ru = urllib.parse.urlparse(migratearv._rootDesc["rootUrl"])
- newuser = arvados.api(host=ru.netloc, token=salted).users().current().execute()
- except arvados.errors.ApiError as e:
- print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
- continue
+ # the user's new home cluster
+ newhomecluster = userhome[0:5]
+ homearv = clusters[newhomecluster]
- try:
- olduser = migratearv.users().get(uuid=old_user_uuid).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Could not retrieve user %s from %s, user may have already been migrated: %s" % (email, old_user_uuid, migratecluster, e))
- continue
+ # create a token for the new user and salt it for the
+ # migration cluster, then use it to access the migration
+ # cluster as the new user once before merging to ensure
+ # the new user is known on that cluster.
+ try:
+ if not args.dry_run:
+ newtok = homearv.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': new_user_uuid}}).execute()
+ else:
+ newtok = {"uuid": "dry-run", "api_token": "12345"}
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not create API token for %s: %s" % (email, new_user_uuid, e))
+ continue
- if not newuser["is_active"]:
- print("(%s) Activating user %s on %s" % (email, new_user_uuid, migratecluster))
+ salted = 'v2/' + newtok["uuid"] + '/' + hmac.new(newtok["api_token"].encode(),
+ msg=migratecluster.encode(),
+ digestmod='sha1').hexdigest()
try:
- migratearv.users().update(uuid=new_user_uuid, body={"is_active": True}).execute()
+ ru = urllib.parse.urlparse(migratearv._rootDesc["rootUrl"])
+ if not args.dry_run:
+ newuser = arvados.api(host=ru.netloc, token=salted, insecure=os.environ.get("ARVADOS_API_HOST_INSECURE")).users().current().execute()
+ else:
+ newuser = {"is_active": True, "username": username}
except arvados.errors.ApiError as e:
- print("(%s) Could not activate user %s on %s: %s" % (email, new_user_uuid, migratecluster, e))
+ print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
continue
- if olduser["is_admin"] and not newuser["is_admin"]:
- print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
- continue
+ try:
+ olduser = migratearv.users().get(uuid=old_user_uuid).execute()
+ except arvados.errors.ApiError as e:
+ if e.resp.status != 404:
+ print("(%s) Could not retrieve user %s from %s, user may have already been migrated: %s" % (email, old_user_uuid, migratecluster, e))
+ continue
+
+ if not newuser["is_active"]:
+ print("(%s) Activating user %s on %s" % (email, new_user_uuid, migratecluster))
+ try:
+ if not args.dry_run:
+ migratearv.users().update(uuid=new_user_uuid, body={"is_active": True}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Could not activate user %s on %s: %s" % (email, new_user_uuid, migratecluster, e))
+ continue
+
+ if olduser["is_admin"] and not newuser["is_admin"]:
+ print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
+ continue
+
+ print("(%s) Migrating %s to %s on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
+
+ try:
+ if not args.dry_run:
+ grp = migratearv.groups().create(body={
+ "owner_uuid": new_user_uuid,
+ "name": "Migrated from %s (%s)" % (email, old_user_uuid),
+ "group_class": "project"
+ }, ensure_unique_name=True).execute()
+ migratearv.users().merge(old_user_uuid=old_user_uuid,
+ new_user_uuid=new_user_uuid,
+ new_owner_uuid=grp["uuid"],
+ redirect_to_new_user=True).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error migrating user: %s" % (email, e))
- print("(%s) Migrating %s to %s on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
-
- try:
- grp = migratearv.groups().create(body={
- "owner_uuid": new_user_uuid,
- "name": "Migrated from %s (%s)" % (email, old_user_uuid),
- "group_class": "project"
- }, ensure_unique_name=True).execute()
- migratearv.users().merge(old_user_uuid=old_user_uuid,
- new_user_uuid=new_user_uuid,
- new_owner_uuid=grp["uuid"],
- redirect_to_new_user=True).execute()
- except arvados.errors.ApiError as e:
- print("(%s) Error migrating user: %s" % (email, e))
+ if newuser['username'] != username:
+ try:
+ if not args.dry_run:
+ conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
+ migratearv.users().update(uuid=new_user_uuid, body={"user": {"username": username}}).execute()
+ except arvados.errors.ApiError as e:
+ print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, new_user_uuid, username, migratecluster, e))
if __name__ == "__main__":
main()
--- /dev/null
+In order to run federation migration integration test:
+
+arvbox must be in the path
+
+arv-federation-migrate should be in the path or the full path supplied
+in the 'fed_migrate' input parameter.
+
+# Create arvbox containers fedbox(1,2,3) for the federation
+$ cwltool --enable-ext arvbox-make-federation.cwl > fed.json
+
+# Configure containers and run tests
+$ cwltool fed-migrate.cwl fed.json
+
+CWL for running the test is generated using cwl-ex:
+
+https://github.com/common-workflow-language/cwl-ex/
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ arvbox_base: Directory
+outputs:
+ arvados_api_hosts:
+ type: string[]
+ outputSource: start/arvados_api_hosts
+ arvados_cluster_ids:
+ type: string[]
+ outputSource: start/arvados_cluster_ids
+ superuser_tokens:
+ type: string[]
+ outputSource: start/superuser_tokens
+ arvbox_containers:
+ type: string[]
+ outputSource: start/arvbox_containers
+requirements:
+ SubworkflowFeatureRequirement: {}
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+steps:
+ start:
+ in:
+ arvbox_base: arvbox_base
+ out: [arvados_api_hosts, arvados_cluster_ids, arvado_api_host_insecure, superuser_tokens, arvbox_containers]
+ run: ../../../cwl/tests/federation/arvbox-make-federation.cwl
--- /dev/null
+import arvados
+import json
+import sys
+
+j = json.load(open(sys.argv[1]))
+
+apiA = arvados.api(host=j["arvados_api_hosts"][0], token=j["superuser_tokens"][0], insecure=True)
+apiB = arvados.api(host=j["arvados_api_hosts"][1], token=j["superuser_tokens"][1], insecure=True)
+apiC = arvados.api(host=j["arvados_api_hosts"][2], token=j["superuser_tokens"][2], insecure=True)
+
+users = apiA.users().list().execute()
+
+assert len(users["items"]) == 10
+
+by_username = {}
+
+for i in range(1, 9):
+ found = False
+ for u in users["items"]:
+ if u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i):
+ found = True
+ by_username[u["username"]] = u["uuid"]
+ assert found
+
+users = apiB.users().list().execute()
+assert len(users["items"]) == 10
+
+for i in range(2, 9):
+ found = False
+ for u in users["items"]:
+ if u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and u["uuid"] == by_username[u["username"]]:
+ found = True
+ assert found
+
+users = apiC.users().list().execute()
+assert len(users["items"]) == 10
+
+for i in range(2, 9):
+ found = False
+ for u in users["items"]:
+ if u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and u["uuid"] == by_username[u["username"]]:
+ found = True
+ assert found
+
+print("Passed checks")
--- /dev/null
+import arvados
+import json
+import sys
+
+j = json.load(open(sys.argv[1]))
+
+apiA = arvados.api(host=j["arvados_api_hosts"][0], token=j["superuser_tokens"][0], insecure=True)
+apiB = arvados.api(host=j["arvados_api_hosts"][1], token=j["superuser_tokens"][1], insecure=True)
+apiC = arvados.api(host=j["arvados_api_hosts"][2], token=j["superuser_tokens"][2], insecure=True)
+
+def maketoken(newtok):
+ return 'v2/' + newtok["uuid"] + '/' + newtok["api_token"]
+
+# case 1
+# user only exists on cluster A
+apiA.users().create(body={"user": {"email": "case1@test"}}).execute()
+
+# case 2
+# user exists on cluster A and has remotes on B and C
+case2 = apiA.users().create(body={"user": {"email": "case2@test"}}).execute()
+newtok = apiA.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case2["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtok), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtok), insecure=True).users().current().execute()
+
+# case 3
+# user only exists on cluster B
+case3 = apiB.users().create(body={"user": {"email": "case3@test"}}).execute()
+
+# case 4
+# user only exists on cluster B and has remotes on A and C
+case4 = apiB.users().create(body={"user": {"email": "case4@test"}}).execute()
+newtok = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case4["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtok), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtok), insecure=True).users().current().execute()
+
+
+# case 5
+# user exists on both cluster A and B
+case5 = apiA.users().create(body={"user": {"email": "case5@test"}}).execute()
+case5 = apiB.users().create(body={"user": {"email": "case5@test"}}).execute()
+
+# case 6
+# user exists on both cluster A and B, with remotes on A, B and C
+case6_A = apiA.users().create(body={"user": {"email": "case6@test"}}).execute()
+newtokA = apiA.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case6_A["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokA), insecure=True).users().current().execute()
+
+case6_B = apiB.users().create(body={"user": {"email": "case6@test"}}).execute()
+newtokB = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case6_B["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
+
+# case 7
+# user exists on both cluster B and A, with remotes on A, B and C
+case7_B = apiB.users().create(body={"user": {"email": "case7@test"}}).execute()
+newtokB = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case7_B["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
+
+case7_A = apiA.users().create(body={"user": {"email": "case7@test"}}).execute()
+newtokA = apiA.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case7_A["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokA), insecure=True).users().current().execute()
+
+# case 8
+# user exists on both cluster B and C, with remotes on A, B and C
+case8_B = apiB.users().create(body={"user": {"email": "case8@test"}}).execute()
+newtokB = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case8_B["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
+
+case8_C = apiC.users().create(body={"user": {"email": "case8@test"}}).execute()
+newtokC = apiC.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case8_C["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokC), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokC), insecure=True).users().current().execute()
--- /dev/null
+#!/usr/bin/env cwl-runner
+$graph:
+ - class: Workflow
+ cwlVersion: v1.0
+ id: '#run_test'
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - default: arv-federation-migrate
+ id: fed_migrate
+ type: string
+ outputs:
+ - id: report3
+ outputSource: main_2/report3
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ MultipleInputFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ SubworkflowFeatureRequirement: {}
+ steps:
+ - id: main_1
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ create_users:
+ default:
+ class: File
+ location: create_users.py
+ superuser_tokens:
+ source: superuser_tokens
+ out:
+ - ar
+ run:
+ arguments:
+ - python
+ - $(inputs.create_users)
+ - _script
+ class: CommandLineTool
+ id: main_1_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: create_users
+ type: File
+ outputs:
+ - id: ar
+ outputBinding:
+ outputEval: $(inputs.arvados_api_hosts)
+ type:
+ items: string
+ type: array
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: main_2
+ in:
+ ar:
+ source: main_1/ar
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ fed_migrate:
+ source: fed_migrate
+ host:
+ valueFrom: '$(inputs.arvados_api_hosts[0])'
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ valueFrom: '$(inputs.superuser_tokens[0])'
+ out:
+ - report
+ - report2
+ - report3
+ - r
+ run:
+ class: Workflow
+ id: main_2_embed
+ inputs:
+ - id: ar
+ type:
+ items: string
+ type: array
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report
+ outputSource: main_2_embed_1/report
+ type: File
+ - id: report2
+ outputSource: main_2_embed_2/report2
+ type: File
+ - id: report3
+ outputSource: main_2_embed_3/report3
+ type: File
+ - id: r
+ outputSource: main_2_embed_4/r
+ type: File
+ requirements:
+ - class: EnvVarRequirement
+ envDef:
+ ARVADOS_API_HOST: $(inputs.host)
+ ARVADOS_API_TOKEN: $(inputs.token)
+ steps:
+ - id: main_2_embed_1
+ in:
+ fed_migrate:
+ source: fed_migrate
+ host:
+ source: host
+ token:
+ source: token
+ out:
+ - report
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--report'
+ - report.csv
+ class: CommandLineTool
+ id: main_2_embed_1_embed
+ inputs:
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_2
+ in:
+ host:
+ source: host
+ report:
+ source: main_2_embed_1/report
+ token:
+ source: token
+ out:
+ - report2
+ run:
+ arguments:
+ - sed
+ - '-E'
+ - 's/,(case[1-8])2?,/,\1,/g'
+ class: CommandLineTool
+ id: main_2_embed_2_embed
+ inputs:
+ - id: report
+ type: File
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report2
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ stdin: $(inputs.report.path)
+ stdout: report.csv
+ - id: main_2_embed_3
+ in:
+ fed_migrate:
+ source: fed_migrate
+ host:
+ source: host
+ report2:
+ source: main_2_embed_2/report2
+ token:
+ source: token
+ out:
+ - report3
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--migrate'
+ - $(inputs.report2)
+ class: CommandLineTool
+ id: main_2_embed_3_embed
+ inputs:
+ - id: report2
+ type: File
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report3
+ outputBinding:
+ outputEval: $(inputs.report2)
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_4
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ check:
+ default:
+ class: File
+ location: check.py
+ host:
+ source: host
+ report3:
+ source: main_2_embed_3/report3
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ source: token
+ out:
+ - r
+ run:
+ arguments:
+ - python
+ - $(inputs.check)
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_4_embed
+ inputs:
+ - id: report3
+ type: File
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: check
+ type: File
+ outputs:
+ - id: r
+ outputBinding:
+ outputEval: $(inputs.report3)
+ type: File
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - arguments:
+ - arvbox
+ - cat
+ - /var/lib/arvados/superuser_token
+ class: CommandLineTool
+ cwlVersion: v1.0
+ id: '#superuser_tok'
+ inputs:
+ - id: container
+ type: string
+ outputs:
+ - id: superuser_token
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: '$(self[0].contents.trim())'
+ type: string
+ requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container)
+ InlineJavascriptRequirement: {}
+ stdout: superuser_token.txt
+ - class: Workflow
+ id: '#main'
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: arvados_cluster_ids
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: arvbox_containers
+ type:
+ items: string
+ type: array
+ - default: arv-federation-migrate
+ id: fed_migrate
+ type: string
+ outputs:
+ - id: supertok
+ outputSource: main_2/supertok
+ type:
+ items: string
+ type: array
+ - id: report
+ outputSource: run_test_3/report3
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ MultipleInputFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ SubworkflowFeatureRequirement: {}
+ steps:
+ - id: main_1
+ in:
+ arvados_cluster_ids:
+ source: arvados_cluster_ids
+ out:
+ - logincluster
+ run:
+ class: ExpressionTool
+ expression: '${return {''logincluster'': (inputs.arvados_cluster_ids[0])};}'
+ inputs:
+ - id: arvados_cluster_ids
+ type:
+ items: string
+ type: array
+ outputs:
+ - id: logincluster
+ type: string
+ - id: main_2
+ in:
+ cluster_id:
+ source: arvados_cluster_ids
+ container:
+ source: arvbox_containers
+ host:
+ source: arvados_api_hosts
+ logincluster:
+ source: main_1/logincluster
+ out:
+ - supertok
+ run:
+ class: Workflow
+ id: main_2_embed
+ inputs:
+ - id: container
+ type: string
+ - id: cluster_id
+ type: string
+ - id: host
+ type: string
+ - id: logincluster
+ type: string
+ outputs:
+ - id: supertok
+ outputSource: superuser_tok_3/superuser_token
+ type: string
+ requirements:
+ - class: EnvVarRequirement
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container)
+ steps:
+ - id: main_2_embed_1
+ in:
+ cluster_id:
+ source: cluster_id
+ container:
+ source: container
+ logincluster:
+ source: logincluster
+ set_login:
+ default:
+ class: File
+ location: set_login.py
+ out:
+ - c
+ run:
+ arguments:
+ - sh
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_1_embed
+ inputs:
+ - id: container
+ type: string
+ - id: cluster_id
+ type: string
+ - id: logincluster
+ type: string
+ - id: set_login
+ type: File
+ outputs:
+ - id: c
+ outputBinding:
+ outputEval: $(inputs.container)
+ type: string
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: >
+ set -x
+
+ docker cp
+ $(inputs.container):/var/lib/arvados/cluster_config.yml.override
+ .
+
+ chmod +w cluster_config.yml.override
+
+ python $(inputs.set_login.path)
+ cluster_config.yml.override $(inputs.cluster_id)
+ $(inputs.logincluster)
+
+ docker cp cluster_config.yml.override
+ $(inputs.container):/var/lib/arvados
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_2
+ in:
+ c:
+ source: main_2_embed_1/c
+ container:
+ source: container
+ host:
+ source: host
+ out:
+ - d
+ run:
+ arguments:
+ - sh
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_2_embed
+ inputs:
+ - id: container
+ type: string
+ - id: host
+ type: string
+ - id: c
+ type: string
+ outputs:
+ - id: d
+ outputBinding:
+ outputEval: $(inputs.c)
+ type: string
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: >
+ set -x
+
+ arvbox hotreset
+
+ while ! curl --fail --insecure --silent
+ https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
+ >/dev/null ; do sleep 3 ; done
+
+ export ARVADOS_API_HOST=$(inputs.host)
+
+ export ARVADOS_API_TOKEN=\$(arvbox cat
+ /var/lib/arvados/superuser_token)
+
+ export ARVADOS_API_HOST_INSECURE=1
+
+ ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat
+ /var/lib/arvados/vm-uuid)
+
+ while ! python -c "import arvados ;
+ arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()"
+ 2>/dev/null ; do sleep 3; done
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: superuser_tok_3
+ in:
+ container:
+ source: container
+ d:
+ source: main_2_embed_2/d
+ out:
+ - superuser_token
+ run: '#superuser_tok'
+ scatter:
+ - container
+ - cluster_id
+ - host
+ scatterMethod: dotproduct
+ - id: run_test_3
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ fed_migrate:
+ source: fed_migrate
+ superuser_tokens:
+ source: main_2/supertok
+ out:
+ - report3
+ run: '#run_test'
+cwlVersion: v1.0
+
--- /dev/null
+import "run-test.cwlex" as run_test
+import "superuser-tok.cwl" as superuser_tok
+
+def workflow main(
+ arvados_api_hosts string[],
+ arvados_cluster_ids string[],
+ superuser_tokens string[],
+ arvbox_containers string[],
+ fed_migrate="arv-federation-migrate"
+) {
+
+ logincluster = run expr (arvados_cluster_ids) string (inputs.arvados_cluster_ids[0])
+
+ scatter arvbox_containers as container,
+ arvados_cluster_ids as cluster_id,
+ arvados_api_hosts as host
+ do run workflow(logincluster)
+ {
+ requirements {
+ EnvVarRequirement {
+ envDef: {
+ ARVBOX_CONTAINER: "$(inputs.container)"
+ }
+ }
+ }
+
+ run tool(container, cluster_id, logincluster, set_login = File("set_login.py")) {
+sh <<<
+set -x
+docker cp $(inputs.container):/var/lib/arvados/cluster_config.yml.override .
+chmod +w cluster_config.yml.override
+python $(inputs.set_login.path) cluster_config.yml.override $(inputs.cluster_id) $(inputs.logincluster)
+docker cp cluster_config.yml.override $(inputs.container):/var/lib/arvados
+>>>
+ return container as c
+ }
+ run tool(container, host, c) {
+sh <<<
+set -x
+arvbox hotreset
+while ! curl --fail --insecure --silent https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest >/dev/null ; do sleep 3 ; done
+export ARVADOS_API_HOST=$(inputs.host)
+export ARVADOS_API_TOKEN=\$(arvbox cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_HOST_INSECURE=1
+ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat /var/lib/arvados/vm-uuid)
+while ! python -c "import arvados ; arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()" 2>/dev/null ; do sleep 3; done
+>>>
+ return c as d
+ }
+ supertok = superuser_tok(container, d)
+ return supertok
+ }
+
+ report = run_test(arvados_api_hosts, superuser_tokens=supertok, fed_migrate)
+ return supertok, report
+}
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env cwl-runner
+class: Workflow
+cwlVersion: v1.0
+id: '#main'
+inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - default: arv-federation-migrate
+ id: fed_migrate
+ type: string
+outputs:
+ - id: report3
+ outputSource: main_2/report3
+ type: File
+requirements:
+ InlineJavascriptRequirement: {}
+ MultipleInputFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ SubworkflowFeatureRequirement: {}
+steps:
+ - id: main_1
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ create_users:
+ default:
+ class: File
+ location: create_users.py
+ superuser_tokens:
+ source: superuser_tokens
+ out:
+ - ar
+ run:
+ arguments:
+ - python
+ - $(inputs.create_users)
+ - _script
+ class: CommandLineTool
+ id: main_1_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: create_users
+ type: File
+ outputs:
+ - id: ar
+ outputBinding:
+ outputEval: $(inputs.arvados_api_hosts)
+ type:
+ items: string
+ type: array
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: main_2
+ in:
+ ar:
+ source: main_1/ar
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ fed_migrate:
+ source: fed_migrate
+ host:
+ valueFrom: '$(inputs.arvados_api_hosts[0])'
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ valueFrom: '$(inputs.superuser_tokens[0])'
+ out:
+ - report
+ - report2
+ - report3
+ - r
+ run:
+ class: Workflow
+ id: main_2_embed
+ inputs:
+ - id: ar
+ type:
+ items: string
+ type: array
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report
+ outputSource: main_2_embed_1/report
+ type: File
+ - id: report2
+ outputSource: main_2_embed_2/report2
+ type: File
+ - id: report3
+ outputSource: main_2_embed_3/report3
+ type: File
+ - id: r
+ outputSource: main_2_embed_4/r
+ type: File
+ requirements:
+ - class: EnvVarRequirement
+ envDef:
+ ARVADOS_API_HOST: $(inputs.host)
+ ARVADOS_API_TOKEN: $(inputs.token)
+ steps:
+ - id: main_2_embed_1
+ in:
+ fed_migrate:
+ source: fed_migrate
+ host:
+ source: host
+ token:
+ source: token
+ out:
+ - report
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--report'
+ - report.csv
+ class: CommandLineTool
+ id: main_2_embed_1_embed
+ inputs:
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_2
+ in:
+ host:
+ source: host
+ report:
+ source: main_2_embed_1/report
+ token:
+ source: token
+ out:
+ - report2
+ run:
+ arguments:
+ - sed
+ - '-E'
+ - 's/,(case[1-8])2?,/,1,/g'
+ class: CommandLineTool
+ id: main_2_embed_2_embed
+ inputs:
+ - id: report
+ type: File
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report2
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ stdin: $(inputs.report)
+ stdout: report.csv
+ - id: main_2_embed_3
+ in:
+ fed_migrate:
+ source: fed_migrate
+ host:
+ source: host
+ report2:
+ source: main_2_embed_2/report2
+ token:
+ source: token
+ out:
+ - report3
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--migrate'
+ - $(inputs.report)
+ class: CommandLineTool
+ id: main_2_embed_3_embed
+ inputs:
+ - id: report2
+ type: File
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report3
+ outputBinding:
+ outputEval: $(inputs.report2)
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_4
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ check:
+ default:
+ class: File
+ location: check.py
+ host:
+ source: host
+ report3:
+ source: main_2_embed_3/report3
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ source: token
+ out:
+ - r
+ run:
+ arguments:
+ - python
+ - $(inputs.check)
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_4_embed
+ inputs:
+ - id: report3
+ type: File
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: check
+ type: File
+ outputs:
+ - id: r
+ outputBinding:
+ outputEval: $(inputs.report3)
+ type: File
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
+
--- /dev/null
+def workflow main(
+ arvados_api_hosts string[],
+ superuser_tokens string[],
+ fed_migrate="arv-federation-migrate"
+) {
+
+ run tool(arvados_api_hosts, superuser_tokens, create_users=File("create_users.py")) {
+ python $(inputs.create_users) <<<
+{
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+}
+>>>
+ return arvados_api_hosts as ar
+ }
+
+ run workflow(ar, arvados_api_hosts, superuser_tokens,
+ fed_migrate,
+ host=$(inputs.arvados_api_hosts[0]),
+ token=$(inputs.superuser_tokens[0])) {
+ requirements {
+ EnvVarRequirement {
+ envDef: {
+ ARVADOS_API_HOST: "$(inputs.host)",
+ ARVADOS_API_TOKEN: "$(inputs.token)"
+ }
+ }
+ }
+
+ run tool(fed_migrate, host, token) {
+ $(inputs.fed_migrate) --report report.csv
+ return File("report.csv") as report
+ }
+
+ run tool(report, host, token) {
+ sed -E 's/,(case[1-8])2?,/,\\1,/g' < $(inputs.report.path) > report.csv
+ return File("report.csv") as report2
+ }
+
+ run tool(report2, fed_migrate, host, token) {
+ $(inputs.fed_migrate) --migrate $(inputs.report2)
+ return report2 as report3
+ }
+
+ run tool(report3, host, token, arvados_api_hosts, superuser_tokens, check=File("check.py")) {
+ python $(inputs.check) <<<
+{
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+}
+>>>
+ return report3 as r
+ }
+ }
+
+ return report3
+}
\ No newline at end of file
--- /dev/null
+import json
+import sys
+
+f = open(sys.argv[1], "r+")
+j = json.load(f)
+j["Clusters"][sys.argv[2]]["Login"] = {"LoginCluster": sys.argv[3]}
+for r in j["Clusters"][sys.argv[2]]["RemoteClusters"]:
+ j["Clusters"][sys.argv[2]]["RemoteClusters"][r]["Insecure"] = True
+f.seek(0)
+json.dump(j, f)
--- /dev/null
+#!/usr/bin/env cwltool
+cwlVersion: v1.0
+class: CommandLineTool
+stdout: superuser_token.txt
+inputs:
+ container: string
+outputs:
+ superuser_token:
+ type: string
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: $(self[0].contents.trim())
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: "$(inputs.container)"
+ InlineJavascriptRequirement: {}
+arguments: [arvbox, cat, /var/lib/arvados/superuser_token]
end
end
end
+
+ discovery[:resources]['configs'] = {
+ methods: {
+ get: {
+ id: "arvados.configs.get",
+ path: "config",
+ httpMethod: "GET",
+ description: "Get public config",
+ parameters: {
+ },
+ parameterOrder: [
+ ],
+ response: {
+ },
+ scopes: [
+ "https://api.curoverse.com/auth/arvados",
+ "https://api.curoverse.com/auth/arvados.readonly"
+ ]
+ },
+ }
+ }
+
Rails.configuration.API.DisabledAPIs.each do |method, _|
ctrl, action = method.to_s.split('.', 2)
discovery[:resources][ctrl][:methods].delete(action.to_sym)
return send_error("cannot move objects into supplied new_owner_uuid: new user does not have write permission", status: 403)
end
- redirect = params[:redirect_to_new_user]
- if @object.uuid[0..4] != Rails.configuration.ClusterID && redirect
- return send_error("cannot merge remote user to other with redirect_to_new_user=true", status: 422)
- end
-
- if !redirect
- return send_error("merge with redirect_to_new_user=false is not yet supported", status: 422)
- end
-
act_as_system_user do
- @object.merge(new_owner_uuid: params[:new_owner_uuid], redirect_to_user_uuid: redirect && new_user.uuid)
+ @object.merge(new_owner_uuid: params[:new_owner_uuid],
+ new_user_uuid: new_user.uuid,
+ redirect_to_new_user: params[:redirect_to_new_user])
end
show
end
uuid_prefix+".arvadosapi.com")
end
- def self.make_http_client
+ def self.make_http_client(uuid_prefix:)
clnt = HTTPClient.new
- if Rails.configuration.TLS.Insecure
+
+ if uuid_prefix && (Rails.configuration.RemoteClusters[uuid_prefix].andand.Insecure ||
+ Rails.configuration.RemoteClusters['*'].andand.Insecure)
clnt.ssl_config.verify_mode = OpenSSL::SSL::VERIFY_NONE
else
# Use system CA certificates
# by a remote cluster when the token absent or expired in our
# database. To begin, we need to ask the cluster that issued
# the token to [re]validate it.
- clnt = ApiClientAuthorization.make_http_client
+ clnt = ApiClientAuthorization.make_http_client(uuid_prefix: token_uuid_prefix)
host = remote_host(uuid_prefix: token_uuid_prefix)
if !host
end
if not (/^#{prefix_match}[A-Za-z][A-Za-z0-9]*$/.match(name))
errors.add(:name,
- "#{errmsg_start} a letter followed by alphanumerics")
+ "#{errmsg_start} a letter followed by alphanumerics, expected pattern '#{prefix_match}[A-Za-z][A-Za-z0-9]*' but was '#{name}'")
false
end
end
end
end
- # Move this user's (i.e., self's) owned items into new_owner_uuid.
- # Also redirect future uses of this account to
- # redirect_to_user_uuid, i.e., when a caller authenticates to this
- # account in the future, the account redirect_to_user_uuid account
- # will be used instead.
+ # Move this user's (i.e., self's) owned items to new_owner_uuid and
+ # new_user_uuid (for things normally owned directly by the user).
+ #
+ # If redirect_auth is true, also reassign auth tokens and ssh keys,
+ # and redirect this account to redirect_to_user_uuid, i.e., when a
+ # caller authenticates to this account in the future, the account
+ # redirect_to_user_uuid account will be used instead.
#
# current_user must have admin privileges, i.e., the caller is
# responsible for checking permission to do this.
- def merge(new_owner_uuid:, redirect_to_user_uuid:)
+ def merge(new_owner_uuid:, new_user_uuid:, redirect_to_new_user:)
raise PermissionDeniedError if !current_user.andand.is_admin
- raise "not implemented" if !redirect_to_user_uuid
+ raise "Missing new_owner_uuid" if !new_owner_uuid
+ raise "Missing new_user_uuid" if !new_user_uuid
transaction(requires_new: true) do
reload
raise "cannot merge an already merged user" if self.redirect_to_user_uuid
- new_user = User.where(uuid: redirect_to_user_uuid).first
+ new_user = User.where(uuid: new_user_uuid).first
raise "user does not exist" if !new_user
raise "cannot merge to an already merged user" if new_user.redirect_to_user_uuid
- # Existing API tokens are updated to authenticate to the new
- # user.
- ApiClientAuthorization.
- where(user_id: id).
- update_all(user_id: new_user.id)
+ # If 'self' is a remote user, don't transfer authorizations
+ # (i.e. ability to access the account) to the new user, because
+ # that gives the remote site the ability to access the 'new'
+ # user account that takes over the 'self' account.
+ #
+ # If 'self' is a local user, it is okay to transfer
+ # authorizations, even if the 'new' user is a remote account,
+ # theq remote site does not gain the ability to access an
+ # account it could not before.
+
+ if redirect_to_new_user and self.uuid[0..4] == Rails.configuration.ClusterID
+ # Existing API tokens and ssh keys are updated to authenticate
+ # to the new user.
+ ApiClientAuthorization.
+ where(user_id: id).
+ update_all(user_id: new_user.id)
+
+ user_updates = [
+ [AuthorizedKey, :owner_uuid],
+ [AuthorizedKey, :authorized_user_uuid],
+ [Link, :owner_uuid],
+ [Link, :tail_uuid],
+ [Link, :head_uuid],
+ ]
+ else
+ # Destroy API tokens and ssh keys associated with the old
+ # user.
+ ApiClientAuthorization.where(user_id: id).destroy_all
+ AuthorizedKey.where(owner_uuid: uuid).destroy_all
+ AuthorizedKey.where(authorized_user_uuid: uuid).destroy_all
+ user_updates = [
+ [Link, :owner_uuid],
+ [Link, :tail_uuid]
+ ]
+ end
# References to the old user UUID in the context of a user ID
# (rather than a "home project" in the project hierarchy) are
# updated to point to the new user.
- [
- [AuthorizedKey, :owner_uuid],
- [AuthorizedKey, :authorized_user_uuid],
- [Repository, :owner_uuid],
- [Link, :owner_uuid],
- [Link, :tail_uuid],
- [Link, :head_uuid],
- ].each do |klass, column|
+ user_updates.each do |klass, column|
klass.where(column => uuid).update_all(column => new_user.uuid)
end
+ # Need to update repository names to new username
+ if username
+ old_repo_name_re = /^#{Regexp.escape(username)}\//
+ Repository.where(:owner_uuid => uuid).each do |repo|
+ repo.owner_uuid = new_user.uuid
+ repo_name_sub = "#{new_user.username}/"
+ name = repo.name.sub(old_repo_name_re, repo_name_sub)
+ while (conflict = Repository.where(:name => name).first) != nil
+ repo_name_sub += "migrated"
+ name = repo.name.sub(old_repo_name_re, repo_name_sub)
+ end
+ repo.name = name
+ repo.save!
+ end
+ end
+
# References to the merged user's "home project" are updated to
# point to new_owner_uuid.
ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
klass.where(owner_uuid: uuid).update_all(owner_uuid: new_owner_uuid)
end
- update_attributes!(redirect_to_user_uuid: new_user.uuid)
+ if redirect_to_new_user
+ update_attributes!(redirect_to_user_uuid: new_user.uuid, username: nil)
+ end
invalidate_permissions_cache
end
end
end
end
- test "refuse to merge with redirect_to_user_uuid=false (not yet supported)" do
+ test "merge with redirect_to_user_uuid=false" do
authorize_with :project_viewer_trustedclient
+ tok = api_client_authorizations(:project_viewer).api_token
post :merge, params: {
new_user_token: api_client_authorizations(:active_trustedclient).api_token,
new_owner_uuid: users(:active).uuid,
redirect_to_new_user: false,
}
- assert_response(422)
+ assert_response(:success)
+ assert_nil(User.unscoped.find_by_uuid(users(:project_viewer).uuid).redirect_to_user_uuid)
+
+ # because redirect_to_new_user=false, token owned by
+ # project_viewer should be deleted
+ auth = ApiClientAuthorization.validate(token: tok)
+ assert_nil(auth)
+ end
+
+ test "merge remote to local as admin" do
+ authorize_with :admin
+
+ remoteuser = User.create!(uuid: "zbbbb-tpzed-remotremotremot")
+ tok = ApiClientAuthorization.create!(user: remoteuser, api_client: api_clients(:untrusted)).api_token
+
+ auth = ApiClientAuthorization.validate(token: tok)
+ assert_not_nil(auth)
+ assert_nil(remoteuser.redirect_to_user_uuid)
+
+ post :merge, params: {
+ new_user_uuid: users(:active).uuid,
+ old_user_uuid: remoteuser.uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ }
+ assert_response(:success)
+ remoteuser.reload
+ assert_equal(users(:active).uuid, remoteuser.redirect_to_user_uuid)
+
+ # token owned by remoteuser should be deleted
+ auth = ApiClientAuthorization.validate(token: tok)
+ assert_nil(auth)
end
test "refuse to merge user into self" do
headers: auth(:active))
assert_response(:success)
assert_equal(users(:project_viewer).uuid, json_response['owner_uuid'])
+ assert_equal("#{users(:project_viewer).username}/foo", json_response['name'])
get('/arvados/v1/groups/' + groups(:aproject).uuid,
params: {},
assert_equal 'barney', json_response['username']
end
+ test 'merge with repository name conflict' do
+ post('/arvados/v1/groups',
+ params: {
+ group: {
+ group_class: 'project',
+ name: "active user's stuff",
+ },
+ },
+ headers: auth(:project_viewer))
+ assert_response(:success)
+ project_uuid = json_response['uuid']
+
+ post('/arvados/v1/repositories/',
+ params: { :repository => { :name => "#{users(:project_viewer).username}/foo", :owner_uuid => users(:project_viewer).uuid } },
+ headers: auth(:project_viewer))
+ assert_response(:success)
+
+ post('/arvados/v1/users/merge',
+ params: {
+ new_user_token: api_client_authorizations(:project_viewer_trustedclient).api_token,
+ new_owner_uuid: project_uuid,
+ redirect_to_new_user: true,
+ },
+ headers: auth(:active_trustedclient))
+ assert_response(:success)
+
+ get('/arvados/v1/repositories/' + repositories(:foo).uuid,
+ params: {},
+ headers: auth(:active))
+ assert_response(:success)
+ assert_equal(users(:project_viewer).uuid, json_response['owner_uuid'])
+ assert_equal("#{users(:project_viewer).username}/migratedfoo", json_response['name'])
+
+ end
+
end
exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados'
;;
+ checkpoint)
+ exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec pg_dump --host=localhost --username=arvados --clean arvados_development > /var/lib/arvados/checkpoint.sql'
+ ;;
+
+ restore)
+ exec docker exec -ti $ARVBOX_CONTAINER bash -c 'PGPASSWORD=$(cat /var/lib/arvados/api_database_pw) exec psql --dbname=arvados_development --host=localhost --username=arvados --quiet --file=/var/lib/arvados/checkpoint.sql'
+ ;;
+
+ hotreset)
+ exec docker exec -i $ARVBOX_CONTAINER /usr/bin/env GEM_HOME=/var/lib/gems /bin/bash - <<EOF
+sv stop api
+sv stop controller
+sv stop websockets
+sv stop keepstore0
+sv stop keepstore1
+sv stop keepproxy
+cd /usr/src/arvados/services/api
+export RAILS_ENV=development
+bundle exec rake db:drop
+rm /var/lib/arvados/api_database_setup
+rm /var/lib/arvados/superuser_token
+rm /var/lib/arvados/keep0-uuid
+rm /var/lib/arvados/keep1-uuid
+rm /var/lib/arvados/keepproxy-uuid
+sv start api
+sv start controller
+sv start websockets
+sv restart keepstore0
+sv restart keepstore1
+sv restart keepproxy
+EOF
+ ;;
+
*)
echo "Arvados-in-a-box https://doc.arvados.org/install/arvbox.html"
echo
echo "build <config> build arvbox Docker image"
echo "reboot <config> stop, build arvbox Docker image, run"
echo "rebuild <config> build arvbox Docker image, no layer cache"
+ echo "checkpoint create database backup"
+ echo "restore restore checkpoint"
echo "reset delete arvbox arvados data (be careful!)"
echo "destroy delete all arvbox code and data (be careful!)"
echo "log <service> tail log of specified service"
exec 2>&1
set -ex -o pipefail
-if [[ -s /etc/arvados/config.yml ]] ; then
+if [[ -s /etc/arvados/config.yml ]] && [[ /var/lib/arvados/cluster_config.yml.override -ot /etc/arvados/config.yml ]] ; then
exit
fi
projects / arvados.git / commitdiff