lib/dispatchcloud/ssh_executor
lib/dispatchcloud/worker
lib/mount
+lib/pam
lib/service
services/api
services/arv-git-httpd
services/keepstore
services/keep-balance
services/login-sync
-services/nodemanager
-services/nodemanager_integration
services/crunch-dispatch-local
services/crunch-dispatch-slurm
services/ws
sdk/cli
-sdk/pam
-sdk/pam:py3
sdk/python
sdk/python:py3
sdk/ruby
( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
|| fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
[[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \
- || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
+ || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
echo Checking dependencies:
echo "locale: ${LANG}"
[[ "$(locale charmap)" = "UTF-8" ]] \
|| fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
echo -n 'libpam pam_appl.h: '
find /usr/include -path '*/security/pam_appl.h' | egrep --max-count=1 . \
- || fatal "No libpam pam_appl.h. Try: apt-get install libpam-dev"
+ || fatal "No libpam pam_appl.h. Try: apt-get install libpam0g-dev"
echo -n 'postgresql: '
psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
echo -n 'phantomjs: '
declare -A only
declare -A testargs
skip[apps/workbench_profile]=1
-# nodemanager_integration tests are not reliable, see #12061.
-skip[services/nodemanager_integration]=1
while [[ -n "$1" ]]
do
fi
if [[ $NEED_SDK_R == false ]]; then
- echo "R SDK not needed, it will not be installed."
+ echo "R SDK not needed, it will not be installed."
fi
checkpidfile() {
. "$VENVDIR/bin/activate"
echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...'
if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
- mkdir -p "$WORKSPACE/services/api/log"
+ mkdir -p "$WORKSPACE/services/api/log"
fi
# Remove empty api.pid file if it exists
if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then
- rm -f "$WORKSPACE/tmp/api.pid"
+ rm -f "$WORKSPACE/tmp/api.pid"
fi
all_services_stopped=
fail=1
python setup.py install
) || fatal "installing PyYAML and sdk/python failed"
- # Preinstall libcloud if using a fork; otherwise nodemanager "pip
- # install" won't pick it up by default.
- if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then
- pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
- || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \
- || fatal "pip install apache-libcloud failed"
- fi
-
# Deactivate Python 2 virtualenv
deactivate
apps/workbench_units | apps/workbench_functionals | apps/workbench_integration)
suite=apps/workbench
;;
- services/nodemanager | services/nodemanager_integration)
- suite=services/nodemanager_suite
- ;;
*)
suite="${1}"
;;
check_arvados_config() {
if [[ "$1" = "env" ]] ; then
- return
+ return
fi
if [[ -z "$ARVADOS_CONFIG" ]] ; then
- # Create config file. The run_test_server script requires PyYAML,
- # so virtualenv needs to be active. Downstream steps like
- # workbench install which require a valid config.yml.
- if [[ ! -s "$VENVDIR/bin/activate" ]] ; then
- install_env
- fi
- . "$VENVDIR/bin/activate"
+ # Create config file. The run_test_server script requires PyYAML,
+ # so virtualenv needs to be active. Downstream steps like
+ # workbench install which require a valid config.yml.
+ if [[ ! -s "$VENVDIR/bin/activate" ]] ; then
+ install_env
+ fi
+ . "$VENVDIR/bin/activate"
cd "$WORKSPACE"
- eval $(python sdk/python/tests/run_test_server.py setup_config)
- deactivate
+ eval $(python sdk/python/tests/run_test_server.py setup_config)
+ deactivate
fi
}
declare -a pythonstuff
pythonstuff=(
- sdk/pam
sdk/python
sdk/python:py3
sdk/cwl:py3
services/dockercleaner:py3
services/fuse
services/fuse:py3
- services/nodemanager
tools/crunchstat-summary
tools/crunchstat-summary:py3
)
(
set -e
cd "$WORKSPACE/doc"
- ARVADOS_API_HOST=qr1hi.arvadosapi.com
+ ARVADOS_API_HOST=pirca.arvadosapi.com
# Make sure python-epydoc is installed or the next line won't
# do much good!
PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
&& "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
}
-test_services/nodemanager_integration() {
- cd "$WORKSPACE/services/nodemanager" \
- && tests/integration_test.py ${testargs[services/nodemanager_integration]}
-}
-
test_apps/workbench_units() {
local TASK="test:units"
cd "$WORKSPACE/apps/workbench" \
do_test sdk/cli
do_test services/login-sync
do_test sdk/java-v2
- do_test services/nodemanager_integration
for p in "${pythonstuff[@]}"
do
dir=${p%:py3}
- Run a workflow using Workbench:
- user/getting_started/workbench.html.textile.liquid
- user/tutorials/tutorial-workflow-workbench.html.textile.liquid
- - user/composer/composer.html.textile.liquid
+ - Working at the Command Line:
+ - user/getting_started/setup-cli.html.textile.liquid
+ - user/reference/api-tokens.html.textile.liquid
+ - user/getting_started/check-environment.html.textile.liquid
- Access an Arvados virtual machine:
- user/getting_started/vm-login-with-webshell.html.textile.liquid
- user/getting_started/ssh-access-unix.html.textile.liquid
- user/getting_started/ssh-access-windows.html.textile.liquid
- - user/getting_started/check-environment.html.textile.liquid
- - user/reference/api-tokens.html.textile.liquid
- Working with data sets:
- user/tutorials/tutorial-keep.html.textile.liquid
- user/tutorials/tutorial-keep-get.html.textile.liquid
- user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid
- user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid
- user/tutorials/tutorial-keep-mount-windows.html.textile.liquid
- - user/topics/keep.html.textile.liquid
- user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid
- user/topics/arv-copy.html.textile.liquid
- - user/topics/storage-classes.html.textile.liquid
- user/topics/collection-versioning.html.textile.liquid
- - Working with git repositories:
- - user/tutorials/add-new-repository.html.textile.liquid
- - user/tutorials/git-arvados-guide.html.textile.liquid
- - Running workflows at the command line:
+ - user/topics/storage-classes.html.textile.liquid
+ - user/topics/keep.html.textile.liquid
+ - Data Analysis with Workflows:
- user/cwl/cwl-runner.html.textile.liquid
- user/cwl/cwl-run-options.html.textile.liquid
- - Develop an Arvados workflow:
- - user/tutorials/intro-crunch.html.textile.liquid
- user/tutorials/writing-cwl-workflow.html.textile.liquid
+ - user/topics/arv-docker.html.textile.liquid
- user/cwl/cwl-style.html.textile.liquid
- - user/cwl/federated-workflows.html.textile.liquid
- user/cwl/cwl-extensions.html.textile.liquid
+ - user/cwl/federated-workflows.html.textile.liquid
- user/cwl/cwl-versions.html.textile.liquid
- - user/topics/arv-docker.html.textile.liquid
+ - Working with git repositories:
+ - user/tutorials/add-new-repository.html.textile.liquid
+ - user/tutorials/git-arvados-guide.html.textile.liquid
- Reference:
- user/topics/link-accounts.html.textile.liquid
- user/reference/cookbook.html.textile.liquid
- sdk/python/example.html.textile.liquid
- sdk/python/python.html.textile.liquid
- sdk/python/arvados-fuse.html.textile.liquid
- - sdk/python/events.html.textile.liquid
+ - sdk/python/arvados-cwl-runner.html.textile.liquid
- sdk/python/cookbook.html.textile.liquid
+ - sdk/python/events.html.textile.liquid
- CLI:
- sdk/cli/install.html.textile.liquid
- sdk/cli/index.html.textile.liquid
- admin/migrating-providers.html.textile.liquid
- user/topics/arvados-sync-groups.html.textile.liquid
- admin/scoped-tokens.html.textile.liquid
+ - admin/token-expiration-policy.html.textile.liquid
- Monitoring:
- admin/logging.html.textile.liquid
- admin/metrics.html.textile.liquid
{% codeblock as python %}
import arvados
api = arvados.api()
- container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+ container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz"
container_request = api.container_requests().get(uuid=container_request_uuid).execute()
print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"])
{% endcodeblock %}
import arvados
import arvados.collection
api = arvados.api()
- container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+ container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz"
container_request = api.container_requests().get(uuid=container_request_uuid).execute()
collection = arvados.collection.CollectionReader(container_request["output_uuid"])
print(collection.open("cwl.output.json").read())
return 'On hold'
else:
return 'Queued'
- elif c['state'] == 'Complete' and c['exit_code'] != 0
+ elif c['state'] == 'Complete' and c['exit_code'] != 0:
return 'Failed'
elif c['state'] == 'Running':
if c['runtime_status'].get('error', None):
elif c['runtime_status'].get('warning', None):
return 'Warning'
return c['state']
- container_request_uuid = 'qr1hi-xvhdp-zzzzzzzzzzzzzzz'
+ container_request_uuid = 'zzzzz-xvhdp-zzzzzzzzzzzzzzz'
print(get_cr_state(container_request_uuid))
{% endcodeblock %}
{% codeblock as python %}
import arvados
api = arvados.api()
- parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+ parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
namefilter = "bwa%" # the "like" filter uses SQL pattern match syntax
container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
parent_container_uuid = container_request["container_uuid"]
{% codeblock as python %}
import arvados
api = arvados.api()
- parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+ parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
namefilter = "bwa%" # the "like" filter uses SQL pattern match syntax
container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
parent_container_uuid = container_request["container_uuid"]
{% codeblock as python %}
import arvados
api = arvados.api()
- parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+ parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
container_request = api.container_requests().get(uuid=parent_request_uuid).execute()
parent_container_uuid = container_request["container_uuid"]
child_requests = api.container_requests().list(filters=[
child_containers = {c["container_uuid"]: c for c in child_requests["items"]}
cancelled_child_containers = api.containers().list(filters=[
["exit_code", "!=", "0"],
- ["uuid", "in", child_containers.keys()]], limit=1000).execute()
+ ["uuid", "in", list(child_containers.keys())]], limit=1000).execute()
for c in cancelled_child_containers["items"]:
print("%s (%s)" % (child_containers[c["uuid"]]["name"], child_containers[c["uuid"]]["uuid"]))
{% endcodeblock %}
import arvados
import arvados.collection
api = arvados.api()
- container_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz"
+ container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz"
container_request = api.container_requests().get(uuid=container_request_uuid).execute()
collection = arvados.collection.CollectionReader(container_request["log_uuid"])
for c in collection:
- print(collection.open(c).read())
+ if isinstance(collection.find(c), arvados.arvfile.ArvadosFile):
+ print(collection.open(c).read())
{% endcodeblock %}
h2(#sharing_link). Create a collection sharing link
import arvados
api = arvados.api()
download="https://your.download.server"
- collection_uuid="qr1hi-4zz18-zzzzzzzzzzzzzzz"
+ collection_uuid="zzzzz-4zz18-zzzzzzzzzzzzzzz"
token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [
"GET /arvados/v1/collections/%s" % collection_uuid,
"GET /arvados/v1/collections/%s/" % collection_uuid,
import arvados
import arvados.collection
api = arvados.api()
- project_uuid = "qr1hi-tpzed-zzzzzzzzzzzzzzz"
- collection_uuids = ["qr1hi-4zz18-aaaaaaaaaaaaaaa", "qr1hi-4zz18-bbbbbbbbbbbbbbb"]
+ project_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz"
+ collection_uuids = ["zzzzz-4zz18-aaaaaaaaaaaaaaa", "zzzzz-4zz18-bbbbbbbbbbbbbbb"]
combined_manifest = ""
for u in collection_uuids:
c = api.collections().get(uuid=u).execute()
import arvados
import arvados.collection
- project_uuid = "qr1hi-j7d0g-zzzzzzzzzzzzzzz"
+ project_uuid = "zzzzz-j7d0g-zzzzzzzzzzzzzzz"
collection_name = "My collection"
filename = "file1.txt"
import arvados
import arvados.collection
- collection_uuid = "qr1hi-4zz18-zzzzzzzzzzzzzzz"
+ collection_uuid = "zzzzz-4zz18-zzzzzzzzzzzzzzz"
filename = "file1.txt"
api = arvados.api()
Workbench2:
InternalURLs: {}
ExternalURL: ""
- Nodemanager:
- InternalURLs: {}
- ExternalURL: "-"
Health:
InternalURLs: {}
ExternalURL: "-"
# Use of this feature is not recommended, if it can be avoided.
ForwardSlashNameSubstitution: ""
+ # Include "folder objects" in S3 ListObjects responses.
+ S3FolderObjects: true
+
# Managed collection properties. At creation time, if the client didn't
# provide the listed keys, they will be automatically populated following
# one of the following behaviors:
ProviderAppID: ""
ProviderAppSecret: ""
+ Test:
+ # Authenticate users listed here in the config file. This
+ # feature is intended to be used in test environments, and
+ # should not be used in production.
+ Enable: false
+ Users:
+ SAMPLE:
+ Email: alice@example.com
+ Password: xyzzy
+
# The cluster ID to delegate the user database. When set,
# logins on this cluster will be redirected to the login cluster
# (login cluster must appear in RemoteClusters with Proxy: true)
# remain valid before it needs to be revalidated.
RemoteTokenRefresh: 5m
+ # How long a client token created from a login flow will be valid without
+ # asking the user to re-login. Example values: 60m, 8h.
+ # Default value zero means tokens don't have expiration.
+ TokenLifetime: 0s
+
Git:
# Path to git or gitolite-shell executable. Each authenticated
# request will execute this program with the single argument "http-backend"
TimeoutShutdown: 10s
# Worker VM image ID.
+ # (aws) AMI identifier
+ # (azure) managed disks: the name of the managed disk image
+ # (azure) shared image gallery: the name of the image definition. Also
+ # see the SharedImageGalleryName and SharedImageGalleryImageVersion fields.
+ # (azure) unmanaged disks (deprecated): the complete URI of the VHD, e.g.
+ # https://xxxxx.blob.core.windows.net/system/Microsoft.Compute/Images/images/xxxxx.vhd
ImageID: ""
# An executable file (located on the dispatcher host) to be
Network: ""
Subnet: ""
- # (azure) Where to store the VM VHD blobs
+ # (azure) managed disks: The resource group where the managed disk
+ # image can be found (if different from ResourceGroup).
+ ImageResourceGroup: ""
+
+ # (azure) shared image gallery: the name of the gallery
+ SharedImageGalleryName: ""
+ # (azure) shared image gallery: the version of the image definition
+ SharedImageGalleryImageVersion: ""
+
+ # (azure) unmanaged disks (deprecated): Where to store the VM VHD blobs
StorageAccount: ""
BlobContainer: ""
# a link to the multi-site search page on a "home" Workbench site.
#
# Example:
- # https://workbench.qr1hi.arvadosapi.com/collections/multisite
+ # https://workbench.zzzzz.arvadosapi.com/collections/multisite
MultiSiteSearch: ""
# Should workbench allow management of local git repositories? Set to false if
Workbench2:
InternalURLs: {}
ExternalURL: ""
- Nodemanager:
- InternalURLs: {}
- ExternalURL: "-"
Health:
InternalURLs: {}
ExternalURL: "-"
# Use of this feature is not recommended, if it can be avoided.
ForwardSlashNameSubstitution: ""
+ # Include "folder objects" in S3 ListObjects responses.
+ S3FolderObjects: true
+
# Managed collection properties. At creation time, if the client didn't
# provide the listed keys, they will be automatically populated following
# one of the following behaviors:
ProviderAppID: ""
ProviderAppSecret: ""
+ Test:
+ # Authenticate users listed here in the config file. This
+ # feature is intended to be used in test environments, and
+ # should not be used in production.
+ Enable: false
+ Users:
+ SAMPLE:
+ Email: alice@example.com
+ Password: xyzzy
+
# The cluster ID to delegate the user database. When set,
# logins on this cluster will be redirected to the login cluster
# (login cluster must appear in RemoteClusters with Proxy: true)
# remain valid before it needs to be revalidated.
RemoteTokenRefresh: 5m
+ # How long a client token created from a login flow will be valid without
+ # asking the user to re-login. Example values: 60m, 8h.
+ # Default value zero means tokens don't have expiration.
+ TokenLifetime: 0s
+
Git:
# Path to git or gitolite-shell executable. Each authenticated
# request will execute this program with the single argument "http-backend"
TimeoutShutdown: 10s
# Worker VM image ID.
+ # (aws) AMI identifier
+ # (azure) managed disks: the name of the managed disk image
+ # (azure) shared image gallery: the name of the image definition. Also
+ # see the SharedImageGalleryName and SharedImageGalleryImageVersion fields.
+ # (azure) unmanaged disks (deprecated): the complete URI of the VHD, e.g.
+ # https://xxxxx.blob.core.windows.net/system/Microsoft.Compute/Images/images/xxxxx.vhd
ImageID: ""
# An executable file (located on the dispatcher host) to be
Network: ""
Subnet: ""
- # (azure) Where to store the VM VHD blobs
+ # (azure) managed disks: The resource group where the managed disk
+ # image can be found (if different from ResourceGroup).
+ ImageResourceGroup: ""
+
+ # (azure) shared image gallery: the name of the gallery
+ SharedImageGalleryName: ""
+ # (azure) shared image gallery: the version of the image definition
+ SharedImageGalleryImageVersion: ""
+
+ # (azure) unmanaged disks (deprecated): Where to store the VM VHD blobs
StorageAccount: ""
BlobContainer: ""
# a link to the multi-site search page on a "home" Workbench site.
#
# Example:
- # https://workbench.qr1hi.arvadosapi.com/collections/multisite
+ # https://workbench.zzzzz.arvadosapi.com/collections/multisite
MultiSiteSearch: ""
# Should workbench allow management of local git repositories? Set to false if
projects / arvados.git / commitdiff