arvados_cluster_ids:
type: string[]
outputSource: start/cluster_id
+ superuser_tokens:
+ type: string[]
+ outputSource: start/superuser_token
acr:
type: string?
outputSource: in_acr
arvado_api_host_insecure:
type: boolean
outputSource: insecure
+ arvbox_containers:
+ type: string[]
+ outputSource: containers
steps:
mkdir:
in:
outputs:
test_user_uuid: string
test_user_token: string
-arguments: [python2, $(inputs.make_user_script)]
\ No newline at end of file
+arguments: [python, $(inputs.make_user_script)]
else:
arv = arvados.api(cache=False)
rh = arv._rootDesc["remoteHosts"]
+ tok = arv.api_client_authorizations().current().execute()
+ token = "v2/%s/%s" % (tok["uuid"], tok["api_token"])
+
for k,v in rh.items():
- arv = arvados.api(host=v, token=os.environ["ARVADOS_API_TOKEN"], cache=False)
+ arv = arvados.api(host=v, token=token, cache=False, insecure=os.environ.get("ARVADOS_API_HOST_INSECURE"))
config = arv.configs().get().execute()
if config["Login"]["LoginCluster"] != "" and loginCluster is None:
loginCluster = config["Login"]["LoginCluster"]
print("Tokens file passed checks")
exit(0)
- if args.report:
- users = []
- for c, arv in clusters.items():
- print("Getting user list from %s" % c)
- ul = arvados.util.list_all(arv.users().list)
- for l in ul:
- if l["uuid"].startswith(c):
- users.append(l)
+ rows = []
+ by_email = {}
- out = csv.writer(open(args.report, "wt"))
+ users = []
+ for c, arv in clusters.items():
+ print("Getting user list from %s" % c)
+ ul = arvados.util.list_all(arv.users().list)
+ for l in ul:
+ if l["uuid"].startswith(c):
+ users.append(l)
- out.writerow(("email", "username", "user uuid", "home cluster"))
+ users = sorted(users, key=lambda u: u["email"]+"::"+(u["username"] or "")+"::"+u["uuid"])
- users = sorted(users, key=lambda u: u["email"]+"::"+(u["username"] or "")+"::"+u["uuid"])
+ accum = []
+ lastemail = None
+ for u in users:
+ if u["uuid"].endswith("-anonymouspublic") or u["uuid"].endswith("-000000000000000"):
+ continue
+ if lastemail == None:
+ lastemail = u["email"]
+ if u["email"] == lastemail:
+ accum.append(u)
+ else:
+ homeuuid = None
+ for a in accum:
+ if homeuuid is None:
+ homeuuid = a["uuid"]
+ if a["uuid"] != homeuuid:
+ homeuuid = ""
+ for a in accum:
+ r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
+ by_email.setdefault(a["email"], [])
+ by_email[a["email"]].append(r)
+ rows.append(r)
+ lastemail = u["email"]
+ accum = [u]
- accum = []
- lastemail = None
- for u in users:
- if u["uuid"].endswith("-anonymouspublic") or u["uuid"].endswith("-000000000000000"):
- continue
- if lastemail == None:
- lastemail = u["email"]
- if u["email"] == lastemail:
- accum.append(u)
- else:
- homeuuid = None
- for a in accum:
- if homeuuid is None:
- homeuuid = a["uuid"]
- if a["uuid"] != homeuuid:
- homeuuid = ""
- for a in accum:
- out.writerow((a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5]))
- lastemail = u["email"]
- accum = [u]
-
- homeuuid = None
- for a in accum:
- if homeuuid is None:
- homeuuid = a["uuid"]
- if a["uuid"] != homeuuid:
- homeuuid = ""
- for a in accum:
- out.writerow((a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5]))
+ homeuuid = None
+ for a in accum:
+ if homeuuid is None:
+ homeuuid = a["uuid"]
+ if a["uuid"] != homeuuid:
+ homeuuid = ""
+ for a in accum:
+ r = (a["email"], a["username"], a["uuid"], loginCluster or homeuuid[0:5])
+ by_email.setdefault(a["email"], [])
+ by_email[a["email"]].append(r)
+ rows.append(r)
+ if args.report:
+ out = csv.writer(open(args.report, "wt"))
+ out.writerow(("email", "username", "user uuid", "home cluster"))
+ for r in rows:
+ out.writerow(r)
print("Wrote %s" % args.report)
+ return
if args.migrate or args.dry_run:
if args.dry_run:
by_email.setdefault(r[0], [])
by_email[r[0]].append(r)
rows.append(r)
+
for r in rows:
email = r[0]
username = r[1]
if old_user_uuid.startswith(userhome):
continue
candidates = []
+ conflict = False
for b in by_email[email]:
if b[2].startswith(userhome):
candidates.append(b)
+ if b[1] != username and b[3] == userhome:
+ print("(%s) Cannot migrate %s, conflicting usernames %s and %s" % (email, old_user_uuid, b[1], username))
+ conflict = True
+ break
+ if conflict:
+ continue
if len(candidates) == 0:
if len(userhome) == 5 and userhome not in clusters:
print("(%s) Cannot migrate %s, unknown home cluster %s (typo?)" % (email, old_user_uuid, userhome))
homearv = clusters[userhome]
user = None
try:
+ conflicts = homearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ homearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
user = homearv.users().create(body={"user": {"email": email, "username": username}}).execute()
except arvados.errors.ApiError as e:
- if "Username" in str(e):
- other = homearv.users().list(filters=[["username", "=", username]]).execute()
- if other['items'] and other['items'][0]['email'] == email:
- conflicting_user = other['items'][0]
- homearv.users().update(uuid=conflicting_user["uuid"], body={"user": {"username": username+"migrate"}}).execute()
- user = homearv.users().create(body={"user": {"email": email, "username": username}}).execute()
- if not user:
- print("(%s) Could not create user: %s" % (email, str(e)))
- continue
-
- candidates.append((email, username, user["uuid"], userhome))
+ print("(%s) Could not create user: %s" % (email, str(e)))
+ continue
+
+ tup = (email, username, user["uuid"], userhome)
+ by_email[email].append(tup)
+ candidates.append(tup)
else:
- candidates.append((email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome))
+ # dry run
+ tup = (email, username, "%s-tpzed-xfakexfakexfake" % (userhome[0:5]), userhome)
+ by_email[email].append(tup)
+ candidates.append(tup)
if len(candidates) > 1:
print("(%s) Multiple users listed to migrate %s to %s, use full uuid" % (email, old_user_uuid, userhome))
continue
try:
ru = urllib.parse.urlparse(migratearv._rootDesc["rootUrl"])
if not args.dry_run:
- newuser = arvados.api(host=ru.netloc, token=salted).users().current().execute()
+ newuser = arvados.api(host=ru.netloc, token=salted, insecure=os.environ.get("ARVADOS_API_HOST_INSECURE")).users().current().execute()
else:
- newuser = {"is_active": True}
+ newuser = {"is_active": True, "username": username}
except arvados.errors.ApiError as e:
print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
continue
print("(%s) Error migrating user: %s" % (email, e))
if newuser['username'] != username:
- print("%s != %s" % (newuser['username'], username))
try:
+ conflicts = migratearv.users().list(filters=[["username", "=", username]]).execute()
+ if conflicts["items"]:
+ migratearv.users().update(uuid=conflicts["items"][0]["uuid"], body={"user": {"username": username+"migrate"}}).execute()
migratearv.users().update(uuid=new_user_uuid, body={"user": {"username": username}}).execute()
except arvados.errors.ApiError as e:
print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, new_user_uuid, username, migratecluster, e))
--- /dev/null
+cwlVersion: v1.0
+class: Workflow
+$namespaces:
+ arv: "http://arvados.org/cwl#"
+ cwltool: "http://commonwl.org/cwltool#"
+inputs:
+ arvbox_base: Directory
+outputs:
+ arvados_api_hosts:
+ type: string[]
+ outputSource: start/arvados_api_hosts
+ arvados_cluster_ids:
+ type: string[]
+ outputSource: start/arvados_cluster_ids
+ superuser_tokens:
+ type: string[]
+ outputSource: start/superuser_tokens
+ arvbox_containers:
+ type: string[]
+ outputSource: start/arvbox_containers
+requirements:
+ SubworkflowFeatureRequirement: {}
+ cwltool:LoadListingRequirement:
+ loadListing: no_listing
+steps:
+ start:
+ in:
+ arvbox_base: arvbox_base
+ out: [arvados_api_hosts, arvados_cluster_ids, arvado_api_host_insecure, superuser_tokens, arvbox_containers]
+ run: ../../../cwl/tests/federation/arvbox-make-federation.cwl
--- /dev/null
+import arvados
+import json
+import sys
+
+j = json.load(open(sys.argv[1]))
+
+apiA = arvados.api(host=j["arvados_api_hosts"][0], token=j["superuser_tokens"][0], insecure=True)
+apiB = arvados.api(host=j["arvados_api_hosts"][1], token=j["superuser_tokens"][1], insecure=True)
+apiC = arvados.api(host=j["arvados_api_hosts"][2], token=j["superuser_tokens"][2], insecure=True)
+
+def maketoken(newtok):
+ return 'v2/' + newtok["uuid"] + '/' + newtok["api_token"]
+
+# case 1
+# user only exists on cluster A
+apiA.users().create(body={"user": {"email": "case1@test"}}).execute()
+
+# case 2
+# user exists on cluster A and has remotes on B and C
+case2 = apiA.users().create(body={"user": {"email": "case2@test"}}).execute()
+newtok = apiA.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case2["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtok), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtok), insecure=True).users().current().execute()
+
+# case 3
+# user only exists on cluster B
+case3 = apiB.users().create(body={"user": {"email": "case3@test"}}).execute()
+
+# case 4
+# user only exists on cluster B and has remotes on A and C
+case4 = apiB.users().create(body={"user": {"email": "case4@test"}}).execute()
+newtok = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case4["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtok), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtok), insecure=True).users().current().execute()
+
+
+# case 5
+# user exists on both cluster A and B
+case5 = apiA.users().create(body={"user": {"email": "case5@test"}}).execute()
+case5 = apiB.users().create(body={"user": {"email": "case5@test"}}).execute()
+
+# case 6
+# user exists on both cluster A and B, with remotes on A, B and C
+case6_A = apiA.users().create(body={"user": {"email": "case6@test"}}).execute()
+newtokA = apiA.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case6_A["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokA), insecure=True).users().current().execute()
+
+case6_B = apiB.users().create(body={"user": {"email": "case6@test"}}).execute()
+newtokB = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case6_B["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
+
+# case 7
+# user exists on both cluster B and A, with remotes on A, B and C
+case7_B = apiB.users().create(body={"user": {"email": "case7@test"}}).execute()
+newtokB = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case7_B["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
+
+case7_A = apiA.users().create(body={"user": {"email": "case7@test"}}).execute()
+newtokA = apiA.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case7_A["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokA), insecure=True).users().current().execute()
+
+# case 8
+# user exists on both cluster B and C, with remotes on A, B and C
+case8_B = apiB.users().create(body={"user": {"email": "case8@test"}}).execute()
+newtokB = apiB.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case8_B["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
+
+case8_C = apiC.users().create(body={"user": {"email": "case8@test"}}).execute()
+newtokC = apiC.api_client_authorizations().create(body={
+ "api_client_authorization": {'owner_uuid': case8_C["uuid"]}}).execute()
+arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokC), insecure=True).users().current().execute()
+arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokC), insecure=True).users().current().execute()
--- /dev/null
+#!/usr/bin/env cwl-runner
+$graph:
+ - class: Workflow
+ cwlVersion: v1.0
+ id: '#run_test'
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - default: arv-federation-migrate
+ id: fed_migrate
+ type: string
+ outputs:
+ - id: report
+ outputSource: main_2/report
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ MultipleInputFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ SubworkflowFeatureRequirement: {}
+ steps:
+ - id: main_1
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ create_users:
+ default:
+ class: File
+ location: create_users.py
+ superuser_tokens:
+ source: superuser_tokens
+ out:
+ - ar
+ run:
+ arguments:
+ - python
+ - $(inputs.create_users)
+ - _script
+ class: CommandLineTool
+ id: main_1_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: create_users
+ type: File
+ outputs:
+ - id: ar
+ outputBinding:
+ outputEval: $(inputs.arvados_api_hosts)
+ type:
+ items: string
+ type: array
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: main_2
+ in:
+ ar:
+ source: main_1/ar
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ fed_migrate:
+ source: fed_migrate
+ host:
+ valueFrom: '$(inputs.arvados_api_hosts[0])'
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ valueFrom: '$(inputs.superuser_tokens[0])'
+ out:
+ - report
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--report'
+ - report.csv
+ class: CommandLineTool
+ id: main_2_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: ar
+ type:
+ items: string
+ type: array
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: report
+ outputBinding:
+ glob: report.csv
+ type: File
+ requirements:
+ - class: EnvVarRequirement
+ envDef:
+ ARVADOS_API_HOST: $(inputs.host)
+ ARVADOS_API_TOKEN: $(inputs.token)
+ - arguments:
+ - arvbox
+ - cat
+ - /var/lib/arvados/superuser_token
+ class: CommandLineTool
+ cwlVersion: v1.0
+ id: '#superuser_tok'
+ inputs:
+ - id: container
+ type: string
+ outputs:
+ - id: superuser_token
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: '$(self[0].contents.trim())'
+ type: string
+ requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container)
+ InlineJavascriptRequirement: {}
+ stdout: superuser_token.txt
+ - class: Workflow
+ id: '#main'
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: arvados_cluster_ids
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: arvbox_containers
+ type:
+ items: string
+ type: array
+ - default: arv-federation-migrate
+ id: fed_migrate
+ type: string
+ outputs:
+ - id: supertok
+ outputSource: main_2/supertok
+ type:
+ items: string
+ type: array
+ - id: report
+ outputSource: run_test_3/report
+ type: File
+ requirements:
+ InlineJavascriptRequirement: {}
+ MultipleInputFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ SubworkflowFeatureRequirement: {}
+ steps:
+ - id: main_1
+ in:
+ arvados_cluster_ids:
+ source: arvados_cluster_ids
+ out:
+ - logincluster
+ run:
+ class: ExpressionTool
+ expression: '${return {''logincluster'': (inputs.arvados_cluster_ids[0])};}'
+ inputs:
+ - id: arvados_cluster_ids
+ type:
+ items: string
+ type: array
+ outputs:
+ - id: logincluster
+ type: string
+ - id: main_2
+ in:
+ cluster_id:
+ source: arvados_cluster_ids
+ container:
+ source: arvbox_containers
+ host:
+ source: arvados_api_hosts
+ logincluster:
+ source: main_1/logincluster
+ out:
+ - supertok
+ run:
+ class: Workflow
+ id: main_2_embed
+ inputs:
+ - id: container
+ type: string
+ - id: cluster_id
+ type: string
+ - id: host
+ type: string
+ - id: logincluster
+ type: string
+ outputs:
+ - id: supertok
+ outputSource: superuser_tok_3/superuser_token
+ type: string
+ requirements:
+ - class: EnvVarRequirement
+ envDef:
+ ARVBOX_CONTAINER: $(inputs.container)
+ steps:
+ - id: main_2_embed_1
+ in:
+ cluster_id:
+ source: cluster_id
+ container:
+ source: container
+ logincluster:
+ source: logincluster
+ set_login:
+ default:
+ class: File
+ location: set_login.py
+ out:
+ - c
+ run:
+ arguments:
+ - sh
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_1_embed
+ inputs:
+ - id: container
+ type: string
+ - id: cluster_id
+ type: string
+ - id: logincluster
+ type: string
+ - id: set_login
+ type: File
+ outputs:
+ - id: c
+ outputBinding:
+ outputEval: $(inputs.container)
+ type: string
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: >
+ set -x
+
+ docker cp
+ $(inputs.container):/var/lib/arvados/cluster_config.yml.override
+ .
+
+ chmod +w cluster_config.yml.override
+
+ python $(inputs.set_login.path)
+ cluster_config.yml.override $(inputs.cluster_id)
+ $(inputs.logincluster)
+
+ docker cp cluster_config.yml.override
+ $(inputs.container):/var/lib/arvados
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: main_2_embed_2
+ in:
+ c:
+ source: main_2_embed_1/c
+ container:
+ source: container
+ host:
+ source: host
+ out:
+ - d
+ run:
+ arguments:
+ - sh
+ - _script
+ class: CommandLineTool
+ id: main_2_embed_2_embed
+ inputs:
+ - id: container
+ type: string
+ - id: host
+ type: string
+ - id: c
+ type: string
+ outputs:
+ - id: d
+ outputBinding:
+ outputEval: $(inputs.c)
+ type: string
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: >
+ set -x
+
+ arvbox hotreset
+
+ while ! curl --fail --insecure --silent
+ https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest
+ >/dev/null ; do sleep 3 ; done
+
+ export ARVADOS_API_HOST=$(inputs.host)
+
+ export ARVADOS_API_TOKEN=\$(arvbox cat
+ /var/lib/arvados/superuser_token)
+
+ export ARVADOS_API_HOST_INSECURE=1
+
+ ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat
+ /var/lib/arvados/vm-uuid)
+
+ while ! python -c "import arvados ;
+ arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()"
+ 2>/dev/null ; do sleep 3; done
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: superuser_tok_3
+ in:
+ container:
+ source: container
+ d:
+ source: main_2_embed_2/d
+ out:
+ - superuser_token
+ run: '#superuser_tok'
+ scatter:
+ - container
+ - cluster_id
+ - host
+ scatterMethod: dotproduct
+ - id: run_test_3
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ fed_migrate:
+ source: fed_migrate
+ superuser_tokens:
+ source: main_2/supertok
+ out:
+ - report
+ run: '#run_test'
+cwlVersion: v1.0
+
--- /dev/null
+import "run-test.cwlex" as run_test
+import "superuser-tok.cwl" as superuser_tok
+
+def workflow main(
+ arvados_api_hosts string[],
+ arvados_cluster_ids string[],
+ superuser_tokens string[],
+ arvbox_containers string[],
+ fed_migrate="arv-federation-migrate"
+) {
+
+ logincluster = run expr (arvados_cluster_ids) string (inputs.arvados_cluster_ids[0])
+
+ scatter arvbox_containers as container,
+ arvados_cluster_ids as cluster_id,
+ arvados_api_hosts as host
+ do run workflow(logincluster)
+ {
+ requirements {
+ EnvVarRequirement {
+ envDef: {
+ ARVBOX_CONTAINER: "$(inputs.container)"
+ }
+ }
+ }
+
+ run tool(container, cluster_id, logincluster, set_login = File("set_login.py")) {
+sh <<<
+set -x
+docker cp $(inputs.container):/var/lib/arvados/cluster_config.yml.override .
+chmod +w cluster_config.yml.override
+python $(inputs.set_login.path) cluster_config.yml.override $(inputs.cluster_id) $(inputs.logincluster)
+docker cp cluster_config.yml.override $(inputs.container):/var/lib/arvados
+>>>
+ return container as c
+ }
+ run tool(container, host, c) {
+sh <<<
+set -x
+arvbox hotreset
+while ! curl --fail --insecure --silent https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest >/dev/null ; do sleep 3 ; done
+export ARVADOS_API_HOST=$(inputs.host)
+export ARVADOS_API_TOKEN=\$(arvbox cat /var/lib/arvados/superuser_token)
+export ARVADOS_API_HOST_INSECURE=1
+ARVADOS_VIRTUAL_MACHINE_UUID=\$(arvbox cat /var/lib/arvados/vm-uuid)
+while ! python -c "import arvados ; arvados.api().virtual_machines().get(uuid='$ARVADOS_VIRTUAL_MACHINE_UUID').execute()" 2>/dev/null ; do sleep 3; done
+>>>
+ return c as d
+ }
+ supertok = superuser_tok(container, d)
+ return supertok
+ }
+
+ report = run_test(arvados_api_hosts, superuser_tokens=supertok, fed_migrate)
+ return supertok, report
+}
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env cwl-runner
+class: Workflow
+cwlVersion: v1.0
+id: '#main'
+inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - default: arv-federation-migrate
+ id: fed_migrate
+ type: string
+outputs:
+ - id: out
+ outputSource: main_2/out
+ type: File
+requirements:
+ InlineJavascriptRequirement: {}
+ MultipleInputFeatureRequirement: {}
+ ScatterFeatureRequirement: {}
+ StepInputExpressionRequirement: {}
+ SubworkflowFeatureRequirement: {}
+steps:
+ - id: main_1
+ in:
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ create_users:
+ default:
+ class: File
+ location: create_users.py
+ superuser_tokens:
+ source: superuser_tokens
+ out:
+ - ar
+ run:
+ arguments:
+ - python
+ - $(inputs.create_users)
+ - _script
+ class: CommandLineTool
+ id: main_1_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: create_users
+ type: File
+ outputs:
+ - id: ar
+ outputBinding:
+ outputEval: $(inputs.arvados_api_hosts)
+ type:
+ items: string
+ type: array
+ requirements:
+ InitialWorkDirRequirement:
+ listing:
+ - entry: |
+ {
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+ }
+ entryname: _script
+ InlineJavascriptRequirement: {}
+ - id: main_2
+ in:
+ ar:
+ source: main_1/ar
+ arvados_api_hosts:
+ source: arvados_api_hosts
+ fed_migrate:
+ source: fed_migrate
+ host:
+ valueFrom: '$(inputs.arvados_api_hosts[0])'
+ superuser_tokens:
+ source: superuser_tokens
+ token:
+ valueFrom: '$(inputs.superuser_tokens[0])'
+ out:
+ - out
+ run:
+ arguments:
+ - $(inputs.fed_migrate)
+ - '--report'
+ - out
+ class: CommandLineTool
+ id: main_2_embed
+ inputs:
+ - id: arvados_api_hosts
+ type:
+ items: string
+ type: array
+ - id: superuser_tokens
+ type:
+ items: string
+ type: array
+ - id: ar
+ type:
+ items: string
+ type: array
+ - id: fed_migrate
+ type: string
+ - id: host
+ type: Any
+ - id: token
+ type: Any
+ outputs:
+ - id: out
+ outputBinding:
+ glob: out
+ type: File
+ requirements:
+ - class: EnvVarRequirement
+ envDef:
+ ARVADOS_API_HOST: $(inputs.host)
+ ARVADOS_API_TOKEN: $(inputs.token)
+
--- /dev/null
+def workflow main(
+ arvados_api_hosts string[],
+ superuser_tokens string[],
+ fed_migrate="arv-federation-migrate"
+) {
+
+ run tool(arvados_api_hosts, superuser_tokens, create_users=File("create_users.py")) {
+ python $(inputs.create_users) <<<
+{
+ "arvados_api_hosts": $(inputs.arvados_api_hosts),
+ "superuser_tokens": $(inputs.superuser_tokens)
+}
+>>>
+ return arvados_api_hosts as ar
+ }
+
+ run tool(arvados_api_hosts, superuser_tokens, ar,
+ fed_migrate,
+ host=$(inputs.arvados_api_hosts[0]),
+ token=$(inputs.superuser_tokens[0])) {
+ requirements {
+ EnvVarRequirement {
+ envDef: {
+ ARVADOS_API_HOST: "$(inputs.host)",
+ ARVADOS_API_TOKEN: "$(inputs.token)"
+ }
+ }
+ }
+ $(inputs.fed_migrate) --report report.csv
+ return File("report.csv") as report
+ }
+
+ return report
+}
\ No newline at end of file
--- /dev/null
+import json
+import sys
+
+f = open(sys.argv[1], "r+")
+j = json.load(f)
+j["Clusters"][sys.argv[2]]["Login"] = {"LoginCluster": sys.argv[3]}
+for r in j["Clusters"][sys.argv[2]]["RemoteClusters"]:
+ j["Clusters"][sys.argv[2]]["RemoteClusters"][r]["Insecure"] = True
+f.seek(0)
+json.dump(j, f)
--- /dev/null
+#!/usr/bin/env cwltool
+cwlVersion: v1.0
+class: CommandLineTool
+stdout: superuser_token.txt
+inputs:
+ container: string
+outputs:
+ superuser_token:
+ type: string
+ outputBinding:
+ glob: superuser_token.txt
+ loadContents: true
+ outputEval: $(self[0].contents.trim())
+requirements:
+ EnvVarRequirement:
+ envDef:
+ ARVBOX_CONTAINER: "$(inputs.container)"
+ InlineJavascriptRequirement: {}
+arguments: [arvbox, cat, /var/lib/arvados/superuser_token]
user_updates = [
[AuthorizedKey, :owner_uuid],
[AuthorizedKey, :authorized_user_uuid],
- [Repository, :owner_uuid],
[Link, :owner_uuid],
[Link, :tail_uuid],
[Link, :head_uuid],
AuthorizedKey.where(owner_uuid: uuid).destroy_all
AuthorizedKey.where(authorized_user_uuid: uuid).destroy_all
user_updates = [
- [Repository, :owner_uuid],
[Link, :owner_uuid],
[Link, :tail_uuid],
[Link, :head_uuid],
klass.where(column => uuid).update_all(column => new_user.uuid)
end
+ # Need to update repository names to new username
+ old_repo_name_re = /^#{Regexp.escape(username)}\//
+ Repository.where(:owner_uuid => uuid).each do |repo|
+ repo.owner_uuid = new_user.uuid
+ repo_name_sub = "#{new_user.username}/"
+ name = repo.name.sub(old_repo_name_re, repo_name_sub)
+ while (conflict = Repository.where(:name => name).first) != nil
+ repo_name_sub += "migrated"
+ name = repo.name.sub(old_repo_name_re, repo_name_sub)
+ end
+ repo.name = name
+ repo.save!
+ end
+
# References to the merged user's "home project" are updated to
# point to new_owner_uuid.
ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
headers: auth(:active))
assert_response(:success)
assert_equal(users(:project_viewer).uuid, json_response['owner_uuid'])
+ assert_equal("#{users(:project_viewer).username}/foo", json_response['name'])
get('/arvados/v1/groups/' + groups(:aproject).uuid,
params: {},
assert_equal 'barney', json_response['username']
end
+ test 'merge with repository name conflict' do
+ post('/arvados/v1/groups',
+ params: {
+ group: {
+ group_class: 'project',
+ name: "active user's stuff",
+ },
+ },
+ headers: auth(:project_viewer))
+ assert_response(:success)
+ project_uuid = json_response['uuid']
+
+ post('/arvados/v1/repositories/',
+ params: { :repository => { :name => "#{users(:project_viewer).username}/foo", :owner_uuid => users(:project_viewer).uuid } },
+ headers: auth(:project_viewer))
+ assert_response(:success)
+
+ post('/arvados/v1/users/merge',
+ params: {
+ new_user_token: api_client_authorizations(:project_viewer_trustedclient).api_token,
+ new_owner_uuid: project_uuid,
+ redirect_to_new_user: true,
+ },
+ headers: auth(:active_trustedclient))
+ assert_response(:success)
+
+ get('/arvados/v1/repositories/' + repositories(:foo).uuid,
+ params: {},
+ headers: auth(:active))
+ assert_response(:success)
+ assert_equal(users(:project_viewer).uuid, json_response['owner_uuid'])
+ assert_equal("#{users(:project_viewer).username}/migratedfoo", json_response['name'])
+
+ end
+
end
projects / arvados.git / commitdiff