# initialize git_internal_dir
# usually /var/lib/arvados/internal.git (set in application.default.yml )
if [ "$APPLICATION_READY" = "1" ]; then
- GIT_INTERNAL_DIR=$($COMMAND_PREFIX bundle exec rake config:check 2>&1 | grep git_internal_dir | awk '{ print $2 }')
+ GIT_INTERNAL_DIR=$($COMMAND_PREFIX bundle exec rake config:dump 2>&1 | grep GitInternalDir | awk '{ print $2 }' |tr -d '"')
if [ ! -e "$GIT_INTERNAL_DIR" ]; then
run_and_report "Creating git_internal_dir '$GIT_INTERNAL_DIR'" \
mkdir -p "$GIT_INTERNAL_DIR"
LICENSE_PACKAGE_TS=20151208015500
if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
- RAILS_PACKAGE_ITERATION=8
+ RAILS_PACKAGE_ITERATION=1
else
RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
fi
gem 'rails-perftest'
gem 'rails-controller-testing'
+gem 'sass-rails'
+
# Install any plugin gems
Dir.glob(File.join(File.dirname(__FILE__), 'lib', '**', "Gemfile")) do |f|
eval(IO.read(f), binding)
faye-websocket (0.10.7)
eventmachine (>= 0.12.0)
websocket-driver (>= 0.5.1)
+ ffi (1.9.25)
globalid (0.4.2)
activesupport (>= 4.2.0)
googleauth (0.8.0)
rake (>= 0.8.7)
thor (>= 0.18.1, < 2.0)
rake (12.3.2)
+ rb-fsevent (0.10.3)
+ rb-inotify (0.9.10)
+ ffi (>= 0.5.0, < 2)
ref (2.0.0)
request_store (1.4.1)
rack (>= 1.4)
rvm-capistrano (1.5.6)
capistrano (~> 2.15.4)
safe_yaml (1.0.5)
+ sass (3.5.5)
+ sass-listen (~> 4.0.0)
+ sass-listen (4.0.0)
+ rb-fsevent (~> 0.9, >= 0.9.4)
+ rb-inotify (~> 0.9, >= 0.9.7)
+ sass-rails (5.0.7)
+ railties (>= 4.0.0, < 6)
+ sass (~> 3.1)
+ sprockets (>= 2.8, < 4.0)
+ sprockets-rails (>= 2.0, < 4.0)
+ tilt (>= 1.1, < 3)
signet (0.11.0)
addressable (~> 2.3)
faraday (~> 0.9)
ref
thor (0.20.3)
thread_safe (0.3.6)
+ tilt (2.0.8)
tzinfo (1.2.5)
thread_safe (~> 0.1)
uglifier (2.7.2)
ruby-prof (~> 0.15.0)
rvm-capistrano
safe_yaml
+ sass-rails
simplecov (~> 0.7.1)
simplecov-rcov
sshkey
* and any sub-directories. You're free to add application-wide styles to this file and they'll appear at
* the top of the compiled file, but it's generally better to create a new file per style scope.
*= require_self
- *= require_tree .
+ *= require_tree .
*/
.contain-align-left {
font-size: 0.8em;
}
img.curoverse-logo {
- width: 221px;
- height: 44px;
+ height: 66px;
}
#intropage {
font-family: Verdana,Arial,sans-serif;
color: #000;
font-weight: bold;
}
-
Rails.cache.fetch 'arvados_v1_rest_discovery' do
Rails.application.eager_load!
remoteHosts = {}
- Rails.configuration.RemoteClusters.each {|k,v| if k != "*" then remoteHosts[k] = v["Host"] end }
+ Rails.configuration.RemoteClusters.each {|k,v| if k != :"*" then remoteHosts[k] = v["Host"] end }
discovery = {
kind: "discovery#restDescription",
discoveryVersion: "v1",
remoteHostsViaDNS: Rails.configuration.RemoteClusters["*"].Proxy,
websocketUrl: Rails.configuration.Services.Websocket.ExternalURL.to_s,
workbenchUrl: Rails.configuration.Services.Workbench1.ExternalURL.to_s,
+ workbench2Url: Rails.configuration.Services.Workbench2.ExternalURL.to_s,
keepWebServiceUrl: Rails.configuration.Services.WebDAV.ExternalURL.to_s,
gitUrl: Rails.configuration.Services.GitHTTP.ExternalURL.to_s,
parameters: {
end
def merge
- if !Thread.current[:api_client].andand.is_trusted
- return send_error("supplied API token is not from a trusted client", status: 403)
- elsif Thread.current[:api_client_authorization].scopes != ['all']
- return send_error("cannot merge with a scoped token", status: 403)
- end
+ if (params[:old_user_uuid] || params[:new_user_uuid])
+ if !current_user.andand.is_admin
+ return send_error("Must be admin to use old_user_uuid/new_user_uuid", status: 403)
+ end
+ if !params[:old_user_uuid] || !params[:new_user_uuid]
+ return send_error("Must supply both old_user_uuid and new_user_uuid", status: 422)
+ end
+ new_user = User.find_by_uuid(params[:new_user_uuid])
+ if !new_user
+ return send_error("User in new_user_uuid not found", status: 422)
+ end
+ @object = User.find_by_uuid(params[:old_user_uuid])
+ if !@object
+ return send_error("User in old_user_uuid not found", status: 422)
+ end
+ else
+ if !Thread.current[:api_client].andand.is_trusted
+ return send_error("supplied API token is not from a trusted client", status: 403)
+ elsif Thread.current[:api_client_authorization].scopes != ['all']
+ return send_error("cannot merge with a scoped token", status: 403)
+ end
- new_auth = ApiClientAuthorization.validate(token: params[:new_user_token])
- if !new_auth
- return send_error("invalid new_user_token", status: 401)
- end
- if !new_auth.api_client.andand.is_trusted
- return send_error("supplied new_user_token is not from a trusted client", status: 403)
- elsif new_auth.scopes != ['all']
- return send_error("supplied new_user_token has restricted scope", status: 403)
+ new_auth = ApiClientAuthorization.validate(token: params[:new_user_token])
+ if !new_auth
+ return send_error("invalid new_user_token", status: 401)
+ end
+
+ if new_auth.user.uuid[0..4] == Rails.configuration.ClusterID
+ if !new_auth.api_client.andand.is_trusted
+ return send_error("supplied new_user_token is not from a trusted client", status: 403)
+ elsif new_auth.scopes != ['all']
+ return send_error("supplied new_user_token has restricted scope", status: 403)
+ end
+ end
+ new_user = new_auth.user
+ @object = current_user
end
- new_user = new_auth.user
- if current_user.uuid == new_user.uuid
+ if @object.uuid == new_user.uuid
return send_error("cannot merge user to self", status: 422)
end
+ if !params[:new_owner_uuid]
+ return send_error("missing new_owner_uuid", status: 422)
+ end
+
if !new_user.can?(write: params[:new_owner_uuid])
return send_error("cannot move objects into supplied new_owner_uuid: new user does not have write permission", status: 403)
end
redirect = params[:redirect_to_new_user]
+ if @object.uuid[0..4] != Rails.configuration.ClusterID && redirect
+ return send_error("cannot merge remote user to other with redirect_to_new_user=true", status: 422)
+ end
+
if !redirect
return send_error("merge with redirect_to_new_user=false is not yet supported", status: 422)
end
- @object = current_user
act_as_system_user do
@object.merge(new_owner_uuid: params[:new_owner_uuid], redirect_to_user_uuid: redirect && new_user.uuid)
end
type: 'string', required: true,
},
new_user_token: {
- type: 'string', required: true,
+ type: 'string', required: false,
},
redirect_to_new_user: {
type: 'boolean', required: false,
},
+ old_user_uuid: {
+ type: 'string', required: false,
+ },
+ new_user_uuid: {
+ type: 'string', required: false,
+ }
}
end
# For the benefit of functional and integration tests:
@user = user
+ if user.uuid[0..4] != Rails.configuration.ClusterID
+ # Actually a remote user
+ # Send them to their home cluster's login
+ rh = Rails.configuration.RemoteClusters[user.uuid[0..4]]
+ remote, return_to_url = params[:return_to].split(',', 2)
+ @remotehomeurl = "#{rh.Scheme || "https"}://#{rh.Host}/login?remote=#{Rails.configuration.ClusterID}&return_to=#{return_to_url}"
+ render
+ return
+ end
+
# prevent ArvadosModel#before_create and _update from throwing
# "unauthorized":
Thread.current[:user] = user
<!DOCTYPE html>
<html>
<head>
- <title>Server</title>
+ <title>Arvados API Server (<%= Rails.configuration.ClusterID %>)</title>
<%= stylesheet_link_tag "application" %>
<%= javascript_include_tag "application" %>
<%= csrf_meta_tags %>
</head>
<body>
<div id="header">
- <div class="apptitle">ARVADOS <span class="beta"><span>BETA</span></span></div>
+ <div class="apptitle">ARVADOS</div>
+ <div>(<%= Rails.configuration.ClusterID %>)</div>
<div style="float:right">
<% if current_user %>
<%= current_user.full_name %>
•
<a class="logout" href="/logout">Log out</a>
<% else %>
- <a class="logout" href="/auth/joshid">Log in</a>
+ <!--<a class="logout" href="/auth/joshid">Log in</a>-->
<% end %>
<% if current_user and session[:real_uid] and session[:switch_back_to] and User.find(session[:real_uid].to_i).verify_userswitch_cookie(session[:switch_back_to]) %>
<p>Sorry, something went wrong logging you in. Please try again.</p>
- <p style="float:right;margin-top:1em">
- <a href="/auth/joshid">Log in here.</a>
- </p>
+ <!--<p style="float:right;margin-top:1em">
+ <a href="/login">Log in here.</a>
+ </p>-->
<div style="clear:both;height:8em"></div>
</div>
--- /dev/null
+<div style="width:30em; margin:2em auto 0 auto">
+ <h1>Login redirect</h1>
+ <p>This login is linked to federated user <b><%= @user.uuid %></b> on cluster <b><%= @user.uuid[0..4] %></b>.</p>
+ <p><a href="<%=@remotehomeurl%>">Click here log in on cluster <%= @user.uuid[0..4] %>.</a></p>
+ <p>After logging in, you will be returned to this cluster (<%=Rails.configuration.ClusterID%>).</p>
+ <p>To avoid seeing this page, choose <b><%= @user.uuid[0..4] %></b> as the cluster that hosts your user account on the Workbench login page.</p>
+</div>
arvcfg.declare_config "Collections.TrashSweepInterval", ActiveSupport::Duration, :trash_sweep_interval
arvcfg.declare_config "Collections.BlobSigningKey", NonemptyString, :blob_signing_key
arvcfg.declare_config "Collections.BlobSigningTTL", Integer, :blob_signature_ttl
-arvcfg.declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest
+arvcfg.declare_config "Collections.BlobSigning", Boolean, :permit_create_collection_with_unsigned_manifest, ->(cfg, k, v) { ConfigLoader.set_cfg cfg, "Collections.BlobSigning", !v }
arvcfg.declare_config "Containers.SupportedDockerImageFormats", Array, :docker_image_formats
arvcfg.declare_config "Containers.LogReuseDecisions", Boolean, :log_reuse_decisions
arvcfg.declare_config "Containers.DefaultKeepCacheRAM", Integer, :container_default_keep_cache_ram
redirect_to_new_user: true,
})
assert_response(:success)
- assert_equal(users(:project_viewer).redirect_to_user_uuid, users(:active).uuid)
+ assert_equal(users(:active).uuid, User.unscoped.find_by_uuid(users(:project_viewer).uuid).redirect_to_user_uuid)
auth = ApiClientAuthorization.validate(token: api_client_authorizations(:project_viewer).api_token)
assert_not_nil(auth)
assert_equal(users(:active).uuid, auth.user.uuid)
end
+
+ test "merge 'project_viewer' account into 'active' account using uuids" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ old_user_uuid: users(:project_viewer).uuid,
+ new_user_uuid: users(:active).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(:success)
+ assert_equal(users(:active).uuid, User.unscoped.find_by_uuid(users(:project_viewer).uuid).redirect_to_user_uuid)
+
+ auth = ApiClientAuthorization.validate(token: api_client_authorizations(:project_viewer).api_token)
+ assert_not_nil(auth)
+ assert_not_nil(auth.user)
+ assert_equal(users(:active).uuid, auth.user.uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied for non-admin" do
+ authorize_with(:active)
+ post(:merge, params: {
+ old_user_uuid: users(:project_viewer).uuid,
+ new_user_uuid: users(:active).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(403)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied missing old_user_uuid" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ new_user_uuid: users(:active).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied missing new_user_uuid" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ old_user_uuid: users(:project_viewer).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied bogus old_user_uuid" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ old_user_uuid: "zzzzz-tpzed-bogusbogusbogus",
+ new_user_uuid: users(:active).uuid,
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
+ test "merge 'project_viewer' account into 'active' account using uuids denied bogus new_user_uuid" do
+ authorize_with(:admin)
+ post(:merge, params: {
+ old_user_uuid: users(:project_viewer).uuid,
+ new_user_uuid: "zzzzz-tpzed-bogusbogusbogus",
+ new_owner_uuid: users(:active).uuid,
+ redirect_to_new_user: true,
+ })
+ assert_response(422)
+ assert_nil(users(:project_viewer).redirect_to_user_uuid)
+ end
+
NON_ADMIN_USER_DATA = ["uuid", "kind", "is_active", "email", "first_name",
"last_name", "username"].sort
fi, err := os.Stat(dev)
if err != nil {
- return giveup("stat %q: %s\n", dev, err)
+ return giveup("stat %q: %s", dev, err)
}
ino := fi.Sys().(*syscall.Stat_t).Ino
n, err := io.Copy(tmpfile, rdr)
v.os.stats.TickOutBytes(uint64(n))
if err != nil {
- log.Printf("%s: writing to %s: %s\n", v, bpath, err)
+ log.Printf("%s: writing to %s: %s", v, bpath, err)
tmpfile.Close()
v.os.Remove(tmpfile.Name())
return err
}
if err := tmpfile.Close(); err != nil {
- log.Printf("closing %s: %s\n", tmpfile.Name(), err)
+ log.Printf("closing %s: %s", tmpfile.Name(), err)
v.os.Remove(tmpfile.Name())
return err
}
if err := v.os.Rename(tmpfile.Name(), bpath); err != nil {
- log.Printf("rename %s %s: %s\n", tmpfile.Name(), bpath, err)
+ log.Printf("rename %s %s: %s", tmpfile.Name(), bpath, err)
return v.os.Remove(tmpfile.Name())
}
return nil
func (v *UnixVolume) Status() *VolumeStatus {
fi, err := v.os.Stat(v.Root)
if err != nil {
- log.Printf("%s: os.Stat: %s\n", v, err)
+ log.Printf("%s: os.Stat: %s", v, err)
return nil
}
devnum := fi.Sys().(*syscall.Stat_t).Dev
var fs syscall.Statfs_t
if err := syscall.Statfs(v.Root, &fs); err != nil {
- log.Printf("%s: statfs: %s\n", v, err)
+ log.Printf("%s: statfs: %s", v, err)
return nil
}
// These calculations match the way df calculates disk usage:
if avail, err := v.FreeDiskSpace(); err == nil {
isFull = avail < MinFreeKilobytes
} else {
- log.Printf("%s: FreeDiskSpace: %s\n", v, err)
+ log.Printf("%s: FreeDiskSpace: %s", v, err)
isFull = false
}
if v.locker == nil {
return nil
}
+ t0 := time.Now()
locked := make(chan struct{})
go func() {
v.locker.Lock()
}()
select {
case <-ctx.Done():
+ log.Printf("%s: client hung up while waiting for Serialize lock (%s)", v, time.Since(t0))
go func() {
<-locked
v.locker.Unlock()
# size class (since libcloud does not provide any consistent API for exposing
# this setting).
# You may also want to define the amount of scratch space (expressed
-# in GB) for Crunch jobs. You can also override Amazon's provided
+# in MB) for Crunch jobs. You can also override Amazon's provided
# data fields (such as price per hour) by setting them here.
#
# Additionally, you can ask for a preemptible instance (AWS's spot instance)
# both spot & reserved versions of the same size, you can do so by renaming
# the Size section and specifying the instance type inside it.
+# 100 GB scratch space
[Size m4.large]
cores = 2
price = 0.126
-scratch = 100
+scratch = 100000
+# 10 GB scratch space
[Size m4.large.spot]
instance_type = m4.large
preemptible = true
cores = 2
price = 0.126
-scratch = 100
+scratch = 10000
+# 200 GB scratch space
[Size m4.xlarge]
cores = 4
price = 0.252
-scratch = 100
+scratch = 200000
'apache-libcloud>=2.3.1.dev1',
'arvados-python-client>=0.1.20170731145219',
'future',
- 'pykka',
+ 'pykka < 2',
'python-daemon',
'setuptools',
'subprocess32>=3.5.1',
;;
root-cert)
- CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.pem
+ CERT=$PWD/${ARVBOX_CONTAINER}-root-cert.crt
if test -n "$1" ; then
CERT="$1"
fi
. /usr/local/lib/arvbox/common.sh
+uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+
if test ! -s /var/lib/arvados/root-cert.pem ; then
# req signing request sub-command
# -new new certificate request
-nodes \
-sha256 \
-x509 \
- -subj "/C=US/ST=MA/O=Arvados testing/OU=arvbox/CN=arvbox testing root CA for ${uuid_prefix}" \
+ -subj "/C=US/ST=MA/O=Arvados testing/OU=arvbox/CN=test root CA for ${uuid_prefix} generated $(date --rfc-3339=seconds)" \
-extensions x509_ext \
-config <(cat /etc/ssl/openssl.cnf \
<(printf "\n[x509_ext]\nbasicConstraints=critical,CA:true,pathlen:0\nkeyUsage=critical,keyCertSign,cRLSign")) \
-new \
-nodes \
-sha256 \
- -subj "/C=US/ST=MA/O=Arvados testing for ${uuid_prefix}/OU=arvbox/CN=localhost" \
+ -subj "/C=US/ST=MA/O=Arvados testing/OU=arvbox/CN=test server cert for ${uuid_prefix} generated $(date --rfc-3339=seconds)" \
-reqexts x509_ext \
-extensions x509_ext \
-config <(cat /etc/ssl/openssl.cnf \
fi
uuid_prefix=$(cat /var/lib/arvados/api_uuid_prefix)
+secret_token=$(cat /var/lib/arvados/api_secret_token)
+blob_signing_key=$(cat /var/lib/arvados/blob_signing_key)
+management_token=$(cat /var/lib/arvados/management_token)
+sso_app_secret=$(cat /var/lib/arvados/sso_app_secret)
+vm_uuid=$(cat /var/lib/arvados/vm-uuid)
database_pw=$(cat /var/lib/arvados/api_database_pw)
if test -s /var/lib/arvados/api_rails_env ; then
cat >/var/lib/arvados/cluster_config.yml <<EOF
Clusters:
${uuid_prefix}:
- NodeProfiles:
+ ManagementToken: $management_token
+ Services:
+ Workbench1:
+ ExternalURL: "https://$localip:${services[workbench]}"
+ Workbench2:
+ ExternalURL: "https://$localip:${services[workbench2-ssl]}"
+ SSO:
+ ExternalURL: "https://$localip:${services[sso]}"
+ Websocket:
+ ExternalURL: "wss://$localip:${services[websockets-ssl]}/websocket"
+ GitSSH:
+ ExternalURL: "ssh://git@$localip:"
+ GitHTTP:
+ ExternalURL: "http://$localip:${services[arv-git-httpd]}/"
+ WebDAV:
+ ExternalURL: "https://$localip:${services[keep-web-ssl]}/"
+ NodeProfiles: # to be deprecated in favor of "Services" section
"*":
arvados-controller:
Listen: ":${services[controller]}" # choose a port
Password: ${database_pw}
DBName: arvados_${database_env}
client_encoding: utf8
+ API:
+ RailsSessionSecretToken: $secret_token
+ Collections:
+ BlobSigningKey: $blob_signing_key
+ DefaultReplication: 1
+ Containers:
+ SupportedDockerImageFormats: ["v2"]
+ Login:
+ ProviderAppSecret: $sso_app_secret
+ ProviderAppID: arvados-server
+ Users:
+ NewUsersAreActive: true
+ AutoAdminFirstUser: true
+ AutoSetupNewUsers: true
+ AutoSetupNewUsersWithVmUUID: $vm_uuid
+ AutoSetupNewUsersWithRepository: true
EOF
/usr/local/lib/arvbox/yml_override.py /var/lib/arvados/cluster_config.yml
cat > /usr/local/bin/crunch-run.sh <<EOF
#!/bin/sh
-exec /usr/local/bin/crunch-run -container-enable-networking=always -container-network-mode=host \$@
+exec /usr/local/bin/crunch-run -container-enable-networking=default -container-network-mode=host \$@
EOF
chmod +x /usr/local/bin/crunch-run.sh
}
EOF
+export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
+export ARVADOS_API_TOKEN=$(cat /var/lib/arvados/superuser_token)
+
+url_prefix="https://$localip:${services[workbench2-ssl]}/"
+
+set +e
+read -rd $'\000' apiclient <<EOF
+{
+ "url_prefix": "$url_prefix",
+ "is_trusted": true
+}
+EOF
+set -e
+
+clientuuid=$(arv --format=uuid api_client list --filters '[["url_prefix", "=", "'$url_prefix'"]]')
+if [[ -n "$clientuuid" ]] ; then
+ arv api_client update --uuid $clientuuid --api-client "$apiclient"
+else
+ arv api_client create --api-client "$apiclient"
+fi
+
export HTTPS=false
# Can't use "yarn start", need to run the dev server script
# directly so that the TERM signal from "sv restart" gets to the
--- /dev/null
+#!/usr/bin/env python
+#
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+#
+
+from __future__ import print_function, absolute_import
+import argparse
+import arvados
+import arvados.util
+import csv
+import sys
+import logging
+
+lglvl = logging.INFO+1
+logging.basicConfig(level=lglvl, format='%(message)s')
+
+"""
+ Given a list of collections missing blocks (as produced by
+keep-balance), produce a report listing affected collections and
+container requests.
+"""
+
+def rerun_request(arv, container_requests_to_rerun, ct):
+ requests = arvados.util.list_all(arv.container_requests().list, filters=[["container_uuid", "=", ct["uuid"]]])
+ for cr in requests:
+ if cr["requesting_container_uuid"]:
+ rerun_request(arv, container_requests_to_rerun, arv.containers().get(uuid=cr["requesting_container_uuid"]).execute())
+ else:
+ container_requests_to_rerun[cr["uuid"]] = cr
+
+def get_owner(arv, owners, record):
+ uuid = record["owner_uuid"]
+ if uuid not in owners:
+ if uuid[6:11] == "tpzed":
+ owners[uuid] = (arv.users().get(uuid=uuid).execute()["full_name"], uuid)
+ else:
+ grp = arv.groups().get(uuid=uuid).execute()
+ _, ou = get_owner(arv, owners, grp)
+ owners[uuid] = (grp["name"], ou)
+ return owners[uuid]
+
+def main():
+ parser = argparse.ArgumentParser(description='Re-run containers associated with missing blocks')
+ parser.add_argument('inp')
+ args = parser.parse_args()
+
+ arv = arvados.api('v1')
+
+ busted_collections = set()
+
+ logging.log(lglvl, "Reading %s", args.inp)
+
+ # Get the list of bad collection PDHs
+ blocksfile = open(args.inp, "rt")
+ for line in blocksfile:
+ # Ignore the first item, that's the block id
+ collections = line.rstrip().split(" ")[1:]
+ for c in collections:
+ busted_collections.add(c)
+
+ out = csv.writer(sys.stdout)
+
+ out.writerow(("collection uuid", "container request uuid", "record name", "modified at", "owner uuid", "owner name", "root owner uuid", "root owner name", "notes"))
+
+ logging.log(lglvl, "Finding collections")
+
+ owners = {}
+ collections_to_delete = {}
+ container_requests_to_rerun = {}
+ # Get containers that produced these collections
+ i = 0
+ for b in busted_collections:
+ if (i % 100) == 0:
+ logging.log(lglvl, "%d/%d", i, len(busted_collections))
+ i += 1
+ collections_to_delete = arvados.util.list_all(arv.collections().list, filters=[["portable_data_hash", "=", b]])
+ for d in collections_to_delete:
+ t = ""
+ if d["properties"].get("type") not in ("output", "log"):
+ t = "\"type\" was '%s', expected one of 'output' or 'log'" % d["properties"].get("type")
+ ou = get_owner(arv, owners, d)
+ out.writerow((d["uuid"], "", d["name"], d["modified_at"], d["owner_uuid"], ou[0], ou[1], owners[ou[1]][0], t))
+
+ maybe_containers_to_rerun = arvados.util.list_all(arv.containers().list, filters=[["output", "=", b]])
+ for ct in maybe_containers_to_rerun:
+ rerun_request(arv, container_requests_to_rerun, ct)
+
+ logging.log(lglvl, "%d/%d", i, len(busted_collections))
+ logging.log(lglvl, "Finding container requests")
+
+ i = 0
+ for _, cr in container_requests_to_rerun.items():
+ if (i % 100) == 0:
+ logging.log(lglvl, "%d/%d", i, len(container_requests_to_rerun))
+ i += 1
+ ou = get_owner(arv, owners, cr)
+ out.writerow(("", cr["uuid"], cr["name"], cr["modified_at"], cr["owner_uuid"], ou[0], ou[1], owners[ou[1]][0], ""))
+
+ logging.log(lglvl, "%d/%d", i, len(container_requests_to_rerun))
+
+if __name__ == "__main__":
+ main()
projects / arvados.git / commitdiff
