From: Peter Amstutz Date: Tue, 25 Aug 2020 18:45:29 +0000 (-0400) Subject: Merge branch '15964-fix-docs' refs #15964 X-Git-Tag: 2.1.0~108 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/a7631a1ccb6e2a6925d00a06562e171c4ce4ea2f?hp=-c Merge branch '15964-fix-docs' refs #15964 Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- a7631a1ccb6e2a6925d00a06562e171c4ce4ea2f diff --combined build/run-tests.sh index bedc95b2db,8ba97a5535..6c697a657b --- a/build/run-tests.sh +++ b/build/run-tests.sh @@@ -91,7 -91,6 +91,7 @@@ lib/dispatchcloud/schedule lib/dispatchcloud/ssh_executor lib/dispatchcloud/worker lib/mount +lib/pam lib/service services/api services/arv-git-httpd @@@ -105,10 -104,14 +105,10 @@@ services/keepprox services/keepstore services/keep-balance services/login-sync -services/nodemanager -services/nodemanager_integration services/crunch-dispatch-local services/crunch-dispatch-slurm services/ws sdk/cli -sdk/pam -sdk/pam:py3 sdk/python sdk/python:py3 sdk/ruby @@@ -195,7 -198,7 +195,7 @@@ sanity_checks() ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)" [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \ - || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" + || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" echo Checking dependencies: echo "locale: ${LANG}" [[ "$(locale charmap)" = "UTF-8" ]] \ @@@ -259,7 -262,7 +259,7 @@@ || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev" echo -n 'libpam pam_appl.h: ' find /usr/include -path '*/security/pam_appl.h' | egrep --max-count=1 . \ - || fatal "No libpam pam_appl.h. Try: apt-get install libpam-dev" + || fatal "No libpam pam_appl.h. Try: apt-get install libpam0g-dev" echo -n 'postgresql: ' psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common" echo -n 'phantomjs: ' @@@ -303,6 -306,8 +303,6 @@@ declare -A ski declare -A only declare -A testargs skip[apps/workbench_profile]=1 -# nodemanager_integration tests are not reliable, see #12061. -skip[services/nodemanager_integration]=1 while [[ -n "$1" ]] do @@@ -373,7 -378,7 +373,7 @@@ if [[ ${skip["sdk/R"]} == 1 && ${skip[" fi if [[ $NEED_SDK_R == false ]]; then - echo "R SDK not needed, it will not be installed." + echo "R SDK not needed, it will not be installed." fi checkpidfile() { @@@ -414,11 -419,11 +414,11 @@@ start_services() . "$VENVDIR/bin/activate" echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then - mkdir -p "$WORKSPACE/services/api/log" + mkdir -p "$WORKSPACE/services/api/log" fi # Remove empty api.pid file if it exists if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then - rm -f "$WORKSPACE/tmp/api.pid" + rm -f "$WORKSPACE/tmp/api.pid" fi all_services_stopped= fail=1 @@@ -663,6 -668,14 +663,6 @@@ install_env() python setup.py install ) || fatal "installing PyYAML and sdk/python failed" - # Preinstall libcloud if using a fork; otherwise nodemanager "pip - # install" won't pick it up by default. - if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then - pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ - || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \ - || fatal "pip install apache-libcloud failed" - fi - # Deactivate Python 2 virtualenv deactivate @@@ -709,6 -722,9 +709,6 @@@ do_test() apps/workbench_units | apps/workbench_functionals | apps/workbench_integration) suite=apps/workbench ;; - services/nodemanager | services/nodemanager_integration) - suite=services/nodemanager_suite - ;; *) suite="${1}" ;; @@@ -817,19 -833,19 +817,19 @@@ do_test_once() check_arvados_config() { if [[ "$1" = "env" ]] ; then - return + return fi if [[ -z "$ARVADOS_CONFIG" ]] ; then - # Create config file. The run_test_server script requires PyYAML, - # so virtualenv needs to be active. Downstream steps like - # workbench install which require a valid config.yml. - if [[ ! -s "$VENVDIR/bin/activate" ]] ; then - install_env - fi - . "$VENVDIR/bin/activate" + # Create config file. The run_test_server script requires PyYAML, + # so virtualenv needs to be active. Downstream steps like + # workbench install which require a valid config.yml. + if [[ ! -s "$VENVDIR/bin/activate" ]] ; then + install_env + fi + . "$VENVDIR/bin/activate" cd "$WORKSPACE" - eval $(python sdk/python/tests/run_test_server.py setup_config) - deactivate + eval $(python sdk/python/tests/run_test_server.py setup_config) + deactivate fi } @@@ -988,12 -1004,14 +988,12 @@@ install_services/api() declare -a pythonstuff pythonstuff=( - sdk/pam sdk/python sdk/python:py3 sdk/cwl:py3 services/dockercleaner:py3 services/fuse services/fuse:py3 - services/nodemanager tools/crunchstat-summary tools/crunchstat-summary:py3 ) @@@ -1012,7 -1030,7 +1012,7 @@@ test_doc() ( set -e cd "$WORKSPACE/doc" - ARVADOS_API_HOST=qr1hi.arvadosapi.com + ARVADOS_API_HOST=pirca.arvadosapi.com # Make sure python-epydoc is installed or the next line won't # do much good! PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST @@@ -1058,6 -1076,11 +1058,6 @@@ test_services/login-sync() && "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]} } -test_services/nodemanager_integration() { - cd "$WORKSPACE/services/nodemanager" \ - && tests/integration_test.py ${testargs[services/nodemanager_integration]} -} - test_apps/workbench_units() { local TASK="test:units" cd "$WORKSPACE/apps/workbench" \ @@@ -1152,6 -1175,7 +1152,6 @@@ test_all() do_test sdk/cli do_test services/login-sync do_test sdk/java-v2 - do_test services/nodemanager_integration for p in "${pythonstuff[@]}" do dir=${p%:py3} diff --combined doc/_config.yml index d1c64c61d8,20b9139986..968ca51fa9 --- a/doc/_config.yml +++ b/doc/_config.yml @@@ -27,38 -27,37 +27,37 @@@ navbar - Run a workflow using Workbench: - user/getting_started/workbench.html.textile.liquid - user/tutorials/tutorial-workflow-workbench.html.textile.liquid - - user/composer/composer.html.textile.liquid + - Working at the Command Line: + - user/getting_started/setup-cli.html.textile.liquid + - user/reference/api-tokens.html.textile.liquid + - user/getting_started/check-environment.html.textile.liquid - Access an Arvados virtual machine: - user/getting_started/vm-login-with-webshell.html.textile.liquid - user/getting_started/ssh-access-unix.html.textile.liquid - user/getting_started/ssh-access-windows.html.textile.liquid - - user/getting_started/check-environment.html.textile.liquid - - user/reference/api-tokens.html.textile.liquid - Working with data sets: - user/tutorials/tutorial-keep.html.textile.liquid - user/tutorials/tutorial-keep-get.html.textile.liquid - user/tutorials/tutorial-keep-mount-gnu-linux.html.textile.liquid - user/tutorials/tutorial-keep-mount-os-x.html.textile.liquid - user/tutorials/tutorial-keep-mount-windows.html.textile.liquid - - user/topics/keep.html.textile.liquid - user/tutorials/tutorial-keep-collection-lifecycle.html.textile.liquid - user/topics/arv-copy.html.textile.liquid - - user/topics/storage-classes.html.textile.liquid - user/topics/collection-versioning.html.textile.liquid - - Working with git repositories: - - user/tutorials/add-new-repository.html.textile.liquid - - user/tutorials/git-arvados-guide.html.textile.liquid - - Running workflows at the command line: + - user/topics/storage-classes.html.textile.liquid + - user/topics/keep.html.textile.liquid + - Data Analysis with Workflows: - user/cwl/cwl-runner.html.textile.liquid - user/cwl/cwl-run-options.html.textile.liquid - - Develop an Arvados workflow: - - user/tutorials/intro-crunch.html.textile.liquid - user/tutorials/writing-cwl-workflow.html.textile.liquid + - user/topics/arv-docker.html.textile.liquid - user/cwl/cwl-style.html.textile.liquid - - user/cwl/federated-workflows.html.textile.liquid - user/cwl/cwl-extensions.html.textile.liquid + - user/cwl/federated-workflows.html.textile.liquid - user/cwl/cwl-versions.html.textile.liquid - - user/topics/arv-docker.html.textile.liquid + - Working with git repositories: + - user/tutorials/add-new-repository.html.textile.liquid + - user/tutorials/git-arvados-guide.html.textile.liquid - Reference: - user/topics/link-accounts.html.textile.liquid - user/reference/cookbook.html.textile.liquid @@@ -75,8 -74,9 +74,9 @@@ - sdk/python/example.html.textile.liquid - sdk/python/python.html.textile.liquid - sdk/python/arvados-fuse.html.textile.liquid - - sdk/python/events.html.textile.liquid + - sdk/python/arvados-cwl-runner.html.textile.liquid - sdk/python/cookbook.html.textile.liquid + - sdk/python/events.html.textile.liquid - CLI: - sdk/cli/install.html.textile.liquid - sdk/cli/index.html.textile.liquid @@@ -162,7 -162,6 +162,7 @@@ - admin/migrating-providers.html.textile.liquid - user/topics/arvados-sync-groups.html.textile.liquid - admin/scoped-tokens.html.textile.liquid + - admin/token-expiration-policy.html.textile.liquid - Monitoring: - admin/logging.html.textile.liquid - admin/metrics.html.textile.liquid diff --combined doc/sdk/python/cookbook.html.textile.liquid index 75c51ee5a8,ff8b8052e5..82741c3ea6 --- a/doc/sdk/python/cookbook.html.textile.liquid +++ b/doc/sdk/python/cookbook.html.textile.liquid @@@ -47,7 -47,7 +47,7 @@@ h2. Get input of a CWL workflo {% codeblock as python %} import arvados api = arvados.api() - container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz" + container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz" container_request = api.container_requests().get(uuid=container_request_uuid).execute() print(container_request["mounts"]["/var/lib/cwl/cwl.input.json"]) {% endcodeblock %} @@@ -58,7 -58,7 +58,7 @@@ h2. Get output of a CWL workflo import arvados import arvados.collection api = arvados.api() - container_request_uuid="qr1hi-xvhdp-zzzzzzzzzzzzzzz" + container_request_uuid="zzzzz-xvhdp-zzzzzzzzzzzzzzz" container_request = api.container_requests().get(uuid=container_request_uuid).execute() collection = arvados.collection.CollectionReader(container_request["output_uuid"]) print(collection.open("cwl.output.json").read()) @@@ -81,7 -81,7 +81,7 @@@ def get_cr_state(cr_uuid) return 'On hold' else: return 'Queued' - elif c['state'] == 'Complete' and c['exit_code'] != 0 + elif c['state'] == 'Complete' and c['exit_code'] != 0: return 'Failed' elif c['state'] == 'Running': if c['runtime_status'].get('error', None): @@@ -89,7 -89,7 +89,7 @@@ elif c['runtime_status'].get('warning', None): return 'Warning' return c['state'] - container_request_uuid = 'qr1hi-xvhdp-zzzzzzzzzzzzzzz' + container_request_uuid = 'zzzzz-xvhdp-zzzzzzzzzzzzzzz' print(get_cr_state(container_request_uuid)) {% endcodeblock %} @@@ -98,7 -98,7 +98,7 @@@ h2. List input of child request {% codeblock as python %} import arvados api = arvados.api() - parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz" + parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz" namefilter = "bwa%" # the "like" filter uses SQL pattern match syntax container_request = api.container_requests().get(uuid=parent_request_uuid).execute() parent_container_uuid = container_request["container_uuid"] @@@ -117,7 -117,7 +117,7 @@@ h2. List output of child request {% codeblock as python %} import arvados api = arvados.api() - parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz" + parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz" namefilter = "bwa%" # the "like" filter uses SQL pattern match syntax container_request = api.container_requests().get(uuid=parent_request_uuid).execute() parent_container_uuid = container_request["container_uuid"] @@@ -136,7 -136,7 +136,7 @@@ h2. List failed child request {% codeblock as python %} import arvados api = arvados.api() - parent_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz" + parent_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz" container_request = api.container_requests().get(uuid=parent_request_uuid).execute() parent_container_uuid = container_request["container_uuid"] child_requests = api.container_requests().list(filters=[ @@@ -144,7 -144,7 +144,7 @@@ child_containers = {c["container_uuid"]: c for c in child_requests["items"]} cancelled_child_containers = api.containers().list(filters=[ ["exit_code", "!=", "0"], - ["uuid", "in", child_containers.keys()]], limit=1000).execute() + ["uuid", "in", list(child_containers.keys())]], limit=1000).execute() for c in cancelled_child_containers["items"]: print("%s (%s)" % (child_containers[c["uuid"]]["name"], child_containers[c["uuid"]]["uuid"])) {% endcodeblock %} @@@ -155,12 -155,11 +155,12 @@@ h2. Get log of a child reques import arvados import arvados.collection api = arvados.api() - container_request_uuid = "qr1hi-xvhdp-zzzzzzzzzzzzzzz" + container_request_uuid = "zzzzz-xvhdp-zzzzzzzzzzzzzzz" container_request = api.container_requests().get(uuid=container_request_uuid).execute() collection = arvados.collection.CollectionReader(container_request["log_uuid"]) for c in collection: - print(collection.open(c).read()) + if isinstance(collection.find(c), arvados.arvfile.ArvadosFile): + print(collection.open(c).read()) {% endcodeblock %} h2(#sharing_link). Create a collection sharing link @@@ -169,7 -168,7 +169,7 @@@ import arvados api = arvados.api() download="https://your.download.server" - collection_uuid="qr1hi-4zz18-zzzzzzzzzzzzzzz" + collection_uuid="zzzzz-4zz18-zzzzzzzzzzzzzzz" token = api.api_client_authorizations().create(body={"api_client_authorization":{"scopes": [ "GET /arvados/v1/collections/%s" % collection_uuid, "GET /arvados/v1/collections/%s/" % collection_uuid, @@@ -185,8 -184,8 +185,8 @@@ Note, if two collections have files of import arvados import arvados.collection api = arvados.api() - project_uuid = "qr1hi-tpzed-zzzzzzzzzzzzzzz" - collection_uuids = ["qr1hi-4zz18-aaaaaaaaaaaaaaa", "qr1hi-4zz18-bbbbbbbbbbbbbbb"] + project_uuid = "zzzzz-tpzed-zzzzzzzzzzzzzzz" + collection_uuids = ["zzzzz-4zz18-aaaaaaaaaaaaaaa", "zzzzz-4zz18-bbbbbbbbbbbbbbb"] combined_manifest = "" for u in collection_uuids: c = api.collections().get(uuid=u).execute() @@@ -201,7 -200,7 +201,7 @@@ h2. Upload a file into a new collectio import arvados import arvados.collection - project_uuid = "qr1hi-j7d0g-zzzzzzzzzzzzzzz" + project_uuid = "zzzzz-j7d0g-zzzzzzzzzzzzzzz" collection_name = "My collection" filename = "file1.txt" @@@ -223,7 -222,7 +223,7 @@@ h2. Download a file from a collectio import arvados import arvados.collection - collection_uuid = "qr1hi-4zz18-zzzzzzzzzzzzzzz" + collection_uuid = "zzzzz-4zz18-zzzzzzzzzzzzzzz" filename = "file1.txt" api = arvados.api() diff --combined lib/config/config.default.yml index 6c47a068e9,f2e27952e1..80294afaf3 --- a/lib/config/config.default.yml +++ b/lib/config/config.default.yml @@@ -139,6 -139,9 +139,6 @@@ Clusters Workbench2: InternalURLs: {} ExternalURL: "" - Nodemanager: - InternalURLs: {} - ExternalURL: "-" Health: InternalURLs: {} ExternalURL: "-" @@@ -483,9 -486,6 +483,9 @@@ # Use of this feature is not recommended, if it can be avoided. ForwardSlashNameSubstitution: "" + # Include "folder objects" in S3 ListObjects responses. + S3FolderObjects: true + # Managed collection properties. At creation time, if the client didn't # provide the listed keys, they will be automatically populated following # one of the following behaviors: @@@ -689,16 -689,6 +689,16 @@@ ProviderAppID: "" ProviderAppSecret: "" + Test: + # Authenticate users listed here in the config file. This + # feature is intended to be used in test environments, and + # should not be used in production. + Enable: false + Users: + SAMPLE: + Email: alice@example.com + Password: xyzzy + # The cluster ID to delegate the user database. When set, # logins on this cluster will be redirected to the login cluster # (login cluster must appear in RemoteClusters with Proxy: true) @@@ -708,11 -698,6 +708,11 @@@ # remain valid before it needs to be revalidated. RemoteTokenRefresh: 5m + # How long a client token created from a login flow will be valid without + # asking the user to re-login. Example values: 60m, 8h. + # Default value zero means tokens don't have expiration. + TokenLifetime: 0s + Git: # Path to git or gitolite-shell executable. Each authenticated # request will execute this program with the single argument "http-backend" @@@ -967,12 -952,6 +967,12 @@@ TimeoutShutdown: 10s # Worker VM image ID. + # (aws) AMI identifier + # (azure) managed disks: the name of the managed disk image + # (azure) shared image gallery: the name of the image definition. Also + # see the SharedImageGalleryName and SharedImageGalleryImageVersion fields. + # (azure) unmanaged disks (deprecated): the complete URI of the VHD, e.g. + # https://xxxxx.blob.core.windows.net/system/Microsoft.Compute/Images/images/xxxxx.vhd ImageID: "" # An executable file (located on the dispatcher host) to be @@@ -1041,16 -1020,7 +1041,16 @@@ Network: "" Subnet: "" - # (azure) Where to store the VM VHD blobs + # (azure) managed disks: The resource group where the managed disk + # image can be found (if different from ResourceGroup). + ImageResourceGroup: "" + + # (azure) shared image gallery: the name of the gallery + SharedImageGalleryName: "" + # (azure) shared image gallery: the version of the image definition + SharedImageGalleryImageVersion: "" + + # (azure) unmanaged disks (deprecated): Where to store the VM VHD blobs StorageAccount: "" BlobContainer: "" @@@ -1314,7 -1284,7 +1314,7 @@@ # a link to the multi-site search page on a "home" Workbench site. # # Example: - # https://workbench.qr1hi.arvadosapi.com/collections/multisite + # https://workbench.zzzzz.arvadosapi.com/collections/multisite MultiSiteSearch: "" # Should workbench allow management of local git repositories? Set to false if diff --combined lib/config/generated_config.go index cbb73d97b8,81d402a6a1..57204cf36a --- a/lib/config/generated_config.go +++ b/lib/config/generated_config.go @@@ -145,6 -145,9 +145,6 @@@ Clusters Workbench2: InternalURLs: {} ExternalURL: "" - Nodemanager: - InternalURLs: {} - ExternalURL: "-" Health: InternalURLs: {} ExternalURL: "-" @@@ -489,9 -492,6 +489,9 @@@ # Use of this feature is not recommended, if it can be avoided. ForwardSlashNameSubstitution: "" + # Include "folder objects" in S3 ListObjects responses. + S3FolderObjects: true + # Managed collection properties. At creation time, if the client didn't # provide the listed keys, they will be automatically populated following # one of the following behaviors: @@@ -695,16 -695,6 +695,16 @@@ ProviderAppID: "" ProviderAppSecret: "" + Test: + # Authenticate users listed here in the config file. This + # feature is intended to be used in test environments, and + # should not be used in production. + Enable: false + Users: + SAMPLE: + Email: alice@example.com + Password: xyzzy + # The cluster ID to delegate the user database. When set, # logins on this cluster will be redirected to the login cluster # (login cluster must appear in RemoteClusters with Proxy: true) @@@ -714,11 -704,6 +714,11 @@@ # remain valid before it needs to be revalidated. RemoteTokenRefresh: 5m + # How long a client token created from a login flow will be valid without + # asking the user to re-login. Example values: 60m, 8h. + # Default value zero means tokens don't have expiration. + TokenLifetime: 0s + Git: # Path to git or gitolite-shell executable. Each authenticated # request will execute this program with the single argument "http-backend" @@@ -973,12 -958,6 +973,12 @@@ TimeoutShutdown: 10s # Worker VM image ID. + # (aws) AMI identifier + # (azure) managed disks: the name of the managed disk image + # (azure) shared image gallery: the name of the image definition. Also + # see the SharedImageGalleryName and SharedImageGalleryImageVersion fields. + # (azure) unmanaged disks (deprecated): the complete URI of the VHD, e.g. + # https://xxxxx.blob.core.windows.net/system/Microsoft.Compute/Images/images/xxxxx.vhd ImageID: "" # An executable file (located on the dispatcher host) to be @@@ -1047,16 -1026,7 +1047,16 @@@ Network: "" Subnet: "" - # (azure) Where to store the VM VHD blobs + # (azure) managed disks: The resource group where the managed disk + # image can be found (if different from ResourceGroup). + ImageResourceGroup: "" + + # (azure) shared image gallery: the name of the gallery + SharedImageGalleryName: "" + # (azure) shared image gallery: the version of the image definition + SharedImageGalleryImageVersion: "" + + # (azure) unmanaged disks (deprecated): Where to store the VM VHD blobs StorageAccount: "" BlobContainer: "" @@@ -1320,7 -1290,7 +1320,7 @@@ # a link to the multi-site search page on a "home" Workbench site. # # Example: - # https://workbench.qr1hi.arvadosapi.com/collections/multisite + # https://workbench.zzzzz.arvadosapi.com/collections/multisite MultiSiteSearch: "" # Should workbench allow management of local git repositories? Set to false if