multi_json (~> 1.0)
websocket-driver (>= 0.2.0)
public_suffix (4.0.3)
- rack (2.2.2)
+ rack (2.2.3)
rack-mini-profiler (1.0.2)
rack (>= 1.2.0)
rack-test (0.6.3)
show_object_using('active', 'groups', 'aproject', 'A Project')
click_on "Sharing"
click_on "Share with groups"
- good_uuid = api_fixture("groups")["private"]["uuid"]
+ good_uuid = api_fixture("groups")["future_project_viewing_group"]["uuid"]
assert(page.has_selector?(".selectable[data-object-uuid=\"#{good_uuid}\"]"),
"'share with groups' listing missing owned user group")
bad_uuid = api_fixture("groups")["asubproject"]["uuid"]
--upload
If the build and test steps are successful, upload the python
packages to pypi and the gems to rubygems (default: false)
+--ruby <true|false>
+ Build ruby gems (default: true)
+--python <true|false>
+ Build python packages (default: true)
WORKSPACE=path Path to the Arvados source tree to build packages from
TARGET=
UPLOAD=0
+RUBY=1
+PYTHON=1
DEBUG=${ARVADOS_DEBUG:-0}
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,upload,target: \
+ help,debug,ruby:,python:,upload,target: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--target)
TARGET="$2"; shift
;;
+ --ruby)
+ RUBY="$2"; shift
+ if [ "$RUBY" != "true" ] && [ "$RUBY" != "1" ]; then
+ RUBY=0
+ else
+ RUBY=1
+ fi
+ ;;
+ --python)
+ PYTHON="$2"; shift
+ if [ "$PYTHON" != "true" ] && [ "$PYTHON" != "1" ]; then
+ PYTHON=0
+ else
+ PYTHON=1
+ fi
+ ;;
--upload)
UPLOAD=1
;;
debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
debug_echo "Workspace is $WORKSPACE"
+if [ $RUBY -eq 0 ] && [ $PYTHON -eq 0 ]; then
+ echo "Nothing to do!"
+ exit 0
+fi
+
if [[ -f /etc/profile.d/rvm.sh ]]; then
source /etc/profile.d/rvm.sh
GEM="rvm-exec default gem"
debug_echo "umask is" `umask`
-gem_wrapper arvados "$WORKSPACE/sdk/ruby"
-gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
-gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
-
GEM_BUILD_FAILURES=0
-if [ ${#failures[@]} -ne 0 ]; then
- GEM_BUILD_FAILURES=${#failures[@]}
+if [ $RUBY -eq 1 ]; then
+ debug_echo "Building Ruby gems"
+ gem_wrapper arvados "$WORKSPACE/sdk/ruby"
+ gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
+ gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
+ if [ ${#failures[@]} -ne 0 ]; then
+ GEM_BUILD_FAILURES=${#failures[@]}
+ fi
fi
-python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
-python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
-python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
-python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
-python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
-
PYTHON_BUILD_FAILURES=0
-if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
- PYTHON_BUILD_FAILURES=${#failures[@]} - $GEM_BUILD_FAILURES
+if [ $PYTHON -eq 1 ]; then
+ debug_echo "Building Python packages"
+ python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
+ python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
+ python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
+ python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
+ python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
+
+ if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
+ PYTHON_BUILD_FAILURES=$((${#failures[@]} - $GEM_BUILD_FAILURES))
+ fi
fi
-if [[ "$UPLOAD" != 0 ]]; then
+if [ $UPLOAD -ne 0 ]; then
+ echo "Uploading"
- if [[ $DEBUG > 0 ]]; then
+ if [ $DEBUG > 0 ]; then
EXTRA_UPLOAD_FLAGS=" --verbose"
else
EXTRA_UPLOAD_FLAGS=""
fi
- if [[ ! -e "$WORKSPACE/packages" ]]; then
+ if [ ! -e "$WORKSPACE/packages" ]; then
mkdir -p "$WORKSPACE/packages"
fi
- title "Start upload python packages"
- timer_reset
-
- if [ "$PYTHON_BUILD_FAILURES" -eq 0 ]; then
- /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
- else
- echo "Skipping python packages upload, there were errors building the packages"
+ if [ $PYTHON -eq 1 ]; then
+ title "Start upload python packages"
+ timer_reset
+
+ if [ $PYTHON_BUILD_FAILURES -eq 0 ]; then
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
+ else
+ echo "Skipping python packages upload, there were errors building the packages"
+ fi
+ checkexit $? "upload python packages"
+ title "End of upload python packages (`timer`)"
fi
- checkexit $? "upload python packages"
- title "End of upload python packages (`timer`)"
- title "Start upload ruby gems"
- timer_reset
-
- if [ "$GEM_BUILD_FAILURES" -eq 0 ]; then
- /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
- else
- echo "Skipping ruby gem upload, there were errors building the packages"
+ if [ $RUBY -eq 1 ]; then
+ title "Start upload ruby gems"
+ timer_reset
+
+ if [ $GEM_BUILD_FAILURES -eq 0 ]; then
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
+ else
+ echo "Skipping ruby gem upload, there were errors building the packages"
+ fi
+ checkexit $? "upload ruby gems"
+ title "End of upload ruby gems (`timer`)"
fi
- checkexit $? "upload ruby gems"
- title "End of upload ruby gems (`timer`)"
-
fi
exit_cleanly
<div class="releasenotes">
</notextile>
-h2(#master). development master (as of 2020-02-07)
+h2(#master). development master (as of 2020-06-17)
"Upgrading from 2.0.0":#v2_0_0
Keepstore now uses "V4 signatures":https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html by default for S3 requests. If you are using Amazon S3, no action is needed; all regions support V4 signatures. If you are using a different S3-compatible service that does not support V4 signatures, add @V2Signature: true@ to your volume driver parameters to preserve the old behavior. See "configuring S3 object storage":{{site.baseurl}}/install/configure-s3-object-storage.html.
+h3. New permission system constraints
+
+Some constraints on the permission system have been added, in particular @role@ and @project@ group types now have distinct behavior. These constraints were already de-facto imposed by the Workbench UI, so on most installations the only effect of this migration will be to reassign @role@ groups to the system user and create a @can_manage@ permission link for the previous owner.
+
+# The @group_class@ field must be either @role@ or @project@. Invalid group_class are migrated to @role@.
+# A @role@ cannot own things. Anything owned by a role is migrated to a @can_manage@ link and reassigned to the system user.
+# Only @role@ and @user@ can have outgoing permission links. Permission links originating from projects are deleted by the migration.
+# A @role@ is always owned by the system_user. When a group is created, it creates a @can_manage@ link for the object that would have been assigned to @owner_uuid@. Migration adds @can_manage@ links and reassigns roles to the system user. This also has the effect of requiring that all @role@ groups have unique names on the system. If there is a name collision during migration, roles will be renamed to ensure they are unique.
+# A permission link can have the permission level (@name@) updated but not @head_uuid@, @tail_uuid@ or @link_class@.
+
+The @arvados-sync-groups@ tool has been updated to reflect these constraints, so it is important to use the version of @arvados-sync-groups@ that matches the API server version.
+
+Before upgrading, use the following commands to find out which groups and permissions in your database will be automatically modified or deleted during the upgrade.
+
+To determine which groups have invalid @group_class@ (these will be migrated to @role@ groups):
+
+<pre>
+arv group list --filters '[["group_class", "not in", ["project", "role"]]]'
+</pre>
+
+To list all @role@ groups, which will be reassigned to the system user (unless @owner_uuid@ is already the system user):
+
+<pre>
+arv group list --filters '[["group_class", "=", "role"]]'
+</pre>
+
+To list which @project@ groups have outgoing permission links (such links are now invalid and will be deleted by the migration):
+
+<pre>
+for uuid in $(arv link list --filters '[["link_class", "=", "permission"], ["tail_uuid", "like", "%-j7d0g-%"]]' |
+ jq -r .items[].tail_uuid | sort | uniq) ; do
+ arv group list --filters '[["group_class", "=", "project"], ["uuid", "=", "'$uuid'"]]' | jq .items
+done
+</pre>
+
h2(#v2_0_0). v2.0.0 (2020-02-07)
"Upgrading from 1.4":#v1_4_1
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-* There are four levels of permission: *none*, *can_read*, *can_write*, and *can_manage*.
-** *none* is the default state when there are no other permission grants.
-*** the object is not included in any list query response.
-*** direct queries of the object by uuid return 404 Not Found.
-*** Link objects require valid identifiers in @head_uuid@ and @tail_uuid@, so an attempt to create a Link that references an unreadable object will return an error indicating the object is not found.
-** *can_read* grants read-only access to the record. Attempting to update or delete the record returns an error. *can_read* does not allow a reader to see any permission grants on the object except the object's owner_uuid and the reader's own permissions.
-** *can_write* permits changes to the record (but not permission links). *can_write* permits the user to delete the object. *can_write* also implies *can_read*.
-** *can_manage* permits the user to read, create, update and delete permission links whose @head_uuid@ is this object's @uuid@. *can_manage* also implies *can_write* and *can_read*.
+There are four levels of permission: *none*, *can_read*, *can_write*, and *can_manage*.
+
+* *none* is the default state when there are no other permission grants.
+** the object is not included in any list query response.
+** direct queries of the object by uuid return 404 Not Found.
+** Link objects require valid identifiers in @head_uuid@ and @tail_uuid@, so an attempt to create a Link that references an unreadable object will return an error indicating the object is not found.
+* *can_read* grants read-only access to the record. Attempting to update or delete the record returns an error.
+** *can_read* does not allow a reader to see any permission grants on the object except the object's owner_uuid and the reader's own permissions.
+* *can_write* permits changes to the record, including changing ownership and deleting the object.
+** *can_write* cannot read, create, update or delete permission links associated with the object.
+** *can_write* also implies *can_read*.
+* *can_manage* permits the user to read, create, update and delete permission links whose @head_uuid@ is this object's @uuid@.
+** *can_manage* also implies *can_write* and *can_read*.
h2. Ownership
-* All Arvados objects have an @owner_uuid@ field. Valid uuid types for @owner_uuid@ are "User" and "Group".
-* The User or Group specified by @owner_uuid@ has *can_manage* permission on the object.
-** This permission is one way: A User or Group's @owner_uuid@ being equal to @X@ does not imply any permission for that User/Group to read, write, or manage an object whose @uuid@ is equal to @X@.
-* Applications should represent each object as belonging to, or being "inside", the Group/User referenced by its @owner_uuid@.
-** A "project" is a subtype of Group that is treated as a "Project" in Workbench, and as a directory by @arv-mount@.
-** A "role" is a subtype of Group that is treated in Workbench as a group of users who have permissions in common (typically an organizational group).
-* To change the @owner_uuid@ field, it is necessary to have @can_write@ permission on both the current owner and the new owner.
+All Arvados objects have an @owner_uuid@ field. Valid uuid types for @owner_uuid@ are "User" and "Group". For Group, the @group_class@ must be a "project".
+
+The User or Group specified by @owner_uuid@ has *can_manage* permission on the object. This permission is one way: an object that is owned does not get any special permissions on the User or Group that owns it.
+
+To change the @owner_uuid@ field, it is necessary to have @can_write@ permission on both the current owner and the new owner.
h2(#links). Permission links
-A link object with
+A permission link is a link object with:
* @owner_uuid@ of the system user.
* @link_class@ "permission"
* @name@ one of *can_read*, *can_write* or *can_manage*
* @head_uuid@ of some Arvados object
-* @tail_uuid@ of a User or Group
+* @tail_uuid@ of a User or Group. For Group, the @group_class@ must be a "role".
-grants the @name@ permission for @tail_uuid@ accessing @head_uuid@
+This grants the permission in @name@ for @tail_uuid@ accessing @head_uuid@.
-* If a User has *can_manage* permission on some object, this grants permission to read, create, update and delete permission links where the @head_uuid@ is the object under management.
+If a User has *can_manage* permission on some object, the user has the ability to read, create, update and delete permission links with @head_uuid@ of the managed object. In other words, the user has the ability to modify the permission grants on the object.
h3. Transitive permissions
-Permissions can be obtained indirectly through Groups.
-* If a User X *can_read* Group A, and Group A *can_read* Object B, then User X *can_read* Object B.
+Permissions can be obtained indirectly through nested ownership (*can_manage*) or by following multiple permission links.
+
+* If a User X owns project A, and project A owns project B, then User X *can_manage* project B.
+* If a User X *can_read* role A, and role A *can_read* Object B, then User X *can_read* Object B.
* Permissions are narrowed to the least powerful permission on the path.
-** If User X *can_write* Group A, and Group A *can_read* Object B, then User X *can_read* Object B.
-** If User X *can_read* Group A, and Group A *can_write* Object B, then User X *can_read* Object B.
+** If User X *can_write* role A, and role A *can_read* Object B, then User X *can_read* Object B.
+** If User X *can_read* role A, and role A *can_write* Object B, then User X *can_read* Object B.
+
+h2. Projects and Roles
+
+A "project" is a subtype of Group that is displayed as a "Project" in Workbench, and as a directory by @arv-mount@.
+* A project can own things (appear in @owner_uuid@)
+* A project can be owned by a user or another project.
+* The name of a project is unique only among projects with the same owner_uuid.
+* Projects can be targets (@head_uuid@) of permission links, but not origins (@tail_uuid@). Putting a project in a @tail_uuid@ field is an error.
+
+A "role" is a subtype of Group that is treated in Workbench as a group of users who have permissions in common (typically an organizational group).
+* A role cannot own things (cannot appear in @owner_uuid@). Putting a role in an @owner_uuid@ field is an error.
+* All roles are owned by the system user.
+* The name of a role is unique across a single Arvados cluster.
+* Roles can be both targets (@head_uuid@) and origins (@tail_uuid@) of permission links.
+
+h3. Access through Roles
-h2. Group Membership
+A "role" consists of a set of users or other roles that have that role, and a set of permissions (primarily read/write/manage access to projects) the role grants.
-Group membership is determined by whether the group has *can_read* permission on an object. If a group G *can_read* an object A, then we say A is a member of G.
+If there is a permission link stating that user A *can_write* role R, then we say A has role R. This means user A has up to *can_write* access to everything the role has access to.
-For some kinds of groups, like roles, it is natural for users who are members of a group to also have *can_manage* permission on the group, i.e., G *can_read* A and A *can_manage* G ("A can do anything G can do"). However, this is not necessary: A can be a member of a group while being unable to even read it.
+Because permissions are one-way, the links A *can_write* R and B *can_write* R does not imply that user A and B will be able to see each other. For users in a role to see each other, read permission should be added going in the opposite direction: R *can_read* A and R *can_read* B.
+
+If a user needs to be able to manipulate permissions of objects that are accessed through the role (for example, to share project P with a user outside the role), then role R must have *can_manage* permission on project P (R *can_manage* P) and the user must be granted *can_manage* permission on R (A *can_manage* R).
h2. Special cases
-* Log table objects are additionally readable based on whether the User has *can_read* permission on @object_uuid@ (User can access log history about objects it can read). To retain the integrity of the log, the log table should deny all update or delete operations.
-* Permission links where @tail_uuid@ is a User permit @can_read@ on the link by that user. (User can discover her own permission grants.)
-* *can_read* on a Collection grants permission to read the blocks that make up the collection (API server returns signed blocks)
-* If User or Group X *can_FOO* Group A, and Group A *can_manage* User B, then X *can_FOO* _everything that User B can_FOO_.
+Log table objects are additionally readable based on whether the User has *can_read* permission on @object_uuid@ (User can access log history about objects it can read). To retain the integrity of the log, the log table denies all update or delete operations.
+
+Permission links where @tail_uuid@ is a User allow *can_read* on the link record by that user (User can discover her own permission grants.)
+
+At least *can_read* on a Collection grants permission to read the blocks that make up the collection (API server returns signed blocks).
+
+A user can only read a container record if the user has read permission to a container_request with that container_uuid.
+
+*can_read* and *can_write* access on a user grants access to the user record, but not anything owned by the user.
+*can_manage* access to a user grants can_manage access to the user, _and everything owned by that user_ .
+If a user A *can_read* role R, and role R *can_manage* user B, then user A *can_read* user B _and everything owned by that user_ .
h2(#system). System user and group
h2. Anoymous user and group
-An Arvados site may be configured to allow users to browse resources without requiring a login. In this case, permissions for non-logged-in users are associated with the "anonymous" user. To make objects visible to the public, they can be shared with the "anonymous" group. The anonymous user uuid is @{siteprefix}-tpzed-anonymouspublic@. The anonymous group uuid is @{siteprefix}-j7d0g-anonymouspublic@.
+An Arvados site may be configured to allow users to browse resources without requiring a login. In this case, permissions for non-logged-in users are associated with the "anonymous" user. To make objects visible to anyone (both logged-in and non-logged-in users), they can be shared with the "anonymous" role. Note that objects shared with the "anonymous" user will only be visible to non-logged-in users!
+
+The anonymous user uuid is @{siteprefix}-tpzed-anonymouspublic@. The anonymous group uuid is @{siteprefix}-j7d0g-anonymouspublic@.
h2. Example
<a name="Support"></a>
<p><strong>Support and Community</strong></p>
- <p>The <a href="https://gitter.im/arvados/community">arvados community channel</a> at gitter.im is available for live discussion and community support. There is also a <a href="http://lists.arvados.org/mailman/listinfo/arvados">mailing list</a>.
+ <p>Interact with the Arvados community on the <a href="https://forum.arvados.org">Arvados Forum</a>
+ and the <a href="https://gitter.im/arvados/community">arvados/community</a> channel at gitter.im.
</p>
<p>Curii Corporation provides managed Arvados installations as well as commercial support for Arvados. Please contact <a href="mailto:info@curii.com">info@curii.com</a> for more information.</p>
<notextile>
<pre>
-<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'
+<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'</span>
{
...
"api_token":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz",
h2. On the web
-The Arvados Free Sofware project page is located at "https://arvados.org":https://arvados.org . The "Arvados Wiki":https://dev.arvados.org/projects/arvados/wiki is a collaborative site for documenting Arvados and provides an overview of the Arvados Platform and Components. The "Arvados blog":https://dev.arvados.org/projects/arvados/blogs posts articles of interest about Arvados.
+The Arvados Free Sofware project page is located at "https://arvados.org":https://arvados.org . The "Arvados Wiki":https://dev.arvados.org/projects/arvados/wiki is a collaborative site for documenting Arvados and provides an overview of the Arvados Platform and Components.
-h2. Mailing lists
+h2. Forum
-The "Arvados user mailing list":http://lists.arvados.org/mailman/listinfo/arvados is a forum for general discussion, questions, and news about Arvados development. The "Arvados developer mailing list":http://lists.arvados.org/mailman/listinfo/arvados-dev is a forum for more technical discussion, intended for developers and contributers to Arvados.
+The "Arvados Forum":https://forum.arvados.org has topic-based discussion, Q&A and community support.
h2. Chat
-The "arvados community channel":https://gitter.im/arvados/community channel at "gitter.im":https://gitter.im is available for live discussion and support.
+The "arvados/community":https://gitter.im/arvados/community channel at "gitter.im":https://gitter.im is available for live discussion and support.
+
+h2. Mailing list
+
+The "Arvados user mailing list":http://lists.arvados.org/mailman/listinfo/arvados is a low-volume list used mainly to announce new releases of Arvados.
h2. Bug tracking
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The Arvados API token is a secret key that enables the @arv@ command line client to access Arvados with the proper permissions.
+The Arvados API token is a secret key that enables the Arvados command line tools to authenticate themselves.
Access the Arvados Workbench using this link: "{{site.arvados_workbench_host}}/":{{site.arvados_workbench_host}}/ (Replace the hostname portion with the hostname of your local Arvados instance if necessary.)
# SPDX-License-Identifier: Apache-2.0
# Based on Debian Stretch
-FROM debian:stretch-slim
-MAINTAINER Peter Amstutz <peter.amstutz@curii.com>
+FROM debian:buster-slim
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
ENV DEBIAN_FRONTEND noninteractive
# apt.arvados.org
-deb http://apt.arvados.org/ stretch-dev main
+deb http://apt.arvados.org/ buster-dev main
# apt.arvados.org
-deb http://apt.arvados.org/ stretch main
+deb http://apt.arvados.org/ buster main
# apt.arvados.org
-deb http://apt.arvados.org/ stretch-testing main
+deb http://apt.arvados.org/ buster-testing main
# or omitted, pages are processed serially.
BalanceCollectionBuffers: 1000
+ # Maximum time for a rebalancing run. This ensures keep-balance
+ # eventually gives up and retries if, for example, a network
+ # error causes a hung connection that is never closed by the
+ # OS. It should be long enough that it doesn't interrupt a
+ # long-running balancing operation.
+ BalanceTimeout: 6h
+
# Default lifetime for ephemeral collections: 2 weeks. This must not
# be less than BlobSigningTTL.
DefaultTrashLifetime: 336h
"Collections.WebDAVCache": false,
"Collections.BalanceCollectionBatch": false,
"Collections.BalancePeriod": false,
+ "Collections.BalanceTimeout": false,
"Collections.BlobMissingReport": false,
"Collections.BalanceCollectionBuffers": false,
"Containers": true,
# or omitted, pages are processed serially.
BalanceCollectionBuffers: 1000
+ # Maximum time for a rebalancing run. This ensures keep-balance
+ # eventually gives up and retries if, for example, a network
+ # error causes a hung connection that is never closed by the
+ # OS. It should be long enough that it doesn't interrupt a
+ # long-running balancing operation.
+ BalanceTimeout: 6h
+
# Default lifetime for ephemeral collections: 2 weeks. This must not
# be less than BlobSigningTTL.
DefaultTrashLifetime: 336h
cluster.TLS.Insecure = true
cluster.API.MaxItemsPerResponse = 1000
cluster.API.MaxRequestAmplification = 4
+ cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
s.testHandler = &Handler{Cluster: cluster}
PostgreSQL: integrationTestCluster().PostgreSQL,
ForceLegacyAPI14: forceLegacyAPI14,
}
+ s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
s.cluster.TLS.Insecure = true
arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
Header: hdrOut,
Body: reqIn.Body,
}).WithContext(reqIn.Context())
-
- resp, err := client.Do(reqOut)
- return resp, err
+ return client.Do(reqOut)
}
// Copy a response (or error) to the downstream client
// Finds the timestamp of the newest copy of blk on svc. Returns
// errNotFound if blk is not on svc at all.
-func (rcvr recoverer) newestMtime(logger logrus.FieldLogger, blk string, svc arvados.KeepService) (time.Time, error) {
- found, err := svc.Index(rcvr.client, blk)
+func (rcvr recoverer) newestMtime(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService) (time.Time, error) {
+ found, err := svc.Index(ctx, rcvr.client, blk)
if err != nil {
logger.WithError(err).Warn("error getting index")
return time.Time{}, err
// saved. But if the block's timestamp is more recent than blobsigttl,
// keepstore will refuse to trash it even if told to by keep-balance.
func (rcvr recoverer) ensureSafe(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService, blobsigttl time.Duration, blobsigexp time.Time) error {
- if latest, err := rcvr.newestMtime(logger, blk, svc); err != nil {
+ if latest, err := rcvr.newestMtime(ctx, logger, blk, svc); err != nil {
return err
} else if latest.Add(blobsigttl).After(blobsigexp) {
return nil
return fmt.Errorf("error updating timestamp: %s", err)
}
logger.Debug("updated timestamp")
- if latest, err := rcvr.newestMtime(logger, blk, svc); err == errNotFound {
+ if latest, err := rcvr.newestMtime(ctx, logger, blk, svc); err == errNotFound {
return fmt.Errorf("(BUG?) touch succeeded, but then block did not appear in index")
} else if err != nil {
return err
import cwltool.process
import cwltool.argparser
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
+from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing
import arvados
import arvados.config
cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
res10 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.0.yml')
res11 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.1.yml')
- customschema10 = res10.read()
- customschema11 = res11.read()
+ customschema10 = res10.read().decode('utf-8')
+ customschema11 = res11.read().decode('utf-8')
use_custom_schema("v1.0", "http://arvados.org/cwl", customschema10)
use_custom_schema("v1.1.0-dev1", "http://arvados.org/cwl", customschema11)
use_custom_schema("v1.1", "http://arvados.org/cwl", customschema11)
from ._version import __version__
from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing, visit_class
+from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing, visit_class
from cwltool.command_line_tool import compute_checksums
from cwltool.load_tool import load_tool
# file to determine what version of cwltool and schema-salad to
# build.
install_requires=[
- 'cwltool==3.0.20200324120055',
- 'schema-salad==5.0.20200302192450',
+ 'cwltool==3.0.20200530110633',
+ 'schema-salad==6.0.20200601095207',
'arvados-python-client{}'.format(pysdk_dep),
'setuptools',
'ciso8601 >= 2.0.0'
// HTTP headers to add/override in outgoing requests.
SendHeader http.Header
+ // Timeout for requests. NewClientFromConfig and
+ // NewClientFromEnv return a Client with a default 5 minute
+ // timeout. To disable this timeout and rely on each
+ // http.Request's context deadline instead, set Timeout to
+ // zero.
+ Timeout time.Duration
+
dd *DiscoveryDocument
- ctx context.Context
+ defaultRequestID string
}
// The default http.Client used by a Client with Insecure==true and
var InsecureHTTPClient = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
- InsecureSkipVerify: true}},
- Timeout: 5 * time.Minute}
+ InsecureSkipVerify: true}}}
// The default http.Client used by a Client otherwise.
-var DefaultSecureClient = &http.Client{
- Timeout: 5 * time.Minute}
+var DefaultSecureClient = &http.Client{}
// NewClientFromConfig creates a new Client that uses the endpoints in
// the given cluster.
Scheme: ctrlURL.Scheme,
APIHost: ctrlURL.Host,
Insecure: cluster.TLS.Insecure,
+ Timeout: 5 * time.Minute,
}, nil
}
AuthToken: os.Getenv("ARVADOS_API_TOKEN"),
Insecure: insecure,
KeepServiceURIs: svcs,
+ Timeout: 5 * time.Minute,
}
}
}
if req.Header.Get("X-Request-Id") == "" {
- reqid, _ := req.Context().Value(contextKeyRequestID{}).(string)
- if reqid == "" {
- reqid, _ = c.context().Value(contextKeyRequestID{}).(string)
- }
- if reqid == "" {
+ var reqid string
+ if ctxreqid, _ := req.Context().Value(contextKeyRequestID{}).(string); ctxreqid != "" {
+ reqid = ctxreqid
+ } else if c.defaultRequestID != "" {
+ reqid = c.defaultRequestID
+ } else {
reqid = reqIDGen.Next()
}
if req.Header == nil {
req.Header.Set("X-Request-Id", reqid)
}
}
- return c.httpClient().Do(req)
+ var cancel context.CancelFunc
+ if c.Timeout > 0 {
+ ctx := req.Context()
+ ctx, cancel = context.WithDeadline(ctx, time.Now().Add(c.Timeout))
+ req = req.WithContext(ctx)
+ }
+ resp, err := c.httpClient().Do(req)
+ if err == nil && cancel != nil {
+ // We need to call cancel() eventually, but we can't
+ // use "defer cancel()" because the context has to
+ // stay alive until the caller has finished reading
+ // the response body.
+ resp.Body = cancelOnClose{ReadCloser: resp.Body, cancel: cancel}
+ } else if cancel != nil {
+ cancel()
+ }
+ return resp, err
+}
+
+// cancelOnClose calls a provided CancelFunc when its wrapped
+// ReadCloser's Close() method is called.
+type cancelOnClose struct {
+ io.ReadCloser
+ cancel context.CancelFunc
+}
+
+func (coc cancelOnClose) Close() error {
+ err := coc.ReadCloser.Close()
+ coc.cancel()
+ return err
}
func isRedirectStatus(code int) bool {
//
// path must not contain a query string.
func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
- return c.RequestAndDecodeContext(c.context(), dst, method, path, body, params)
+ return c.RequestAndDecodeContext(context.Background(), dst, method, path, body, params)
}
func (c *Client) RequestAndDecodeContext(ctx context.Context, dst interface{}, method, path string, body io.Reader, params interface{}) error {
// header.
func (c *Client) WithRequestID(reqid string) *Client {
cc := *c
- cc.ctx = ContextWithRequestID(cc.context(), reqid)
+ cc.defaultRequestID = reqid
return &cc
}
-func (c *Client) context() context.Context {
- if c.ctx == nil {
- return context.Background()
- }
- return c.ctx
-}
-
func (c *Client) httpClient() *http.Client {
switch {
case c.Client != nil:
BalancePeriod Duration
BalanceCollectionBatch int
BalanceCollectionBuffers int
+ BalanceTimeout Duration
WebDAVCache WebDAVCacheConfig
}
}
// Index returns an unsorted list of blocks at the given mount point.
-func (s *KeepService) IndexMount(c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
- return s.index(c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
+func (s *KeepService) IndexMount(ctx context.Context, c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
+ return s.index(ctx, c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
}
// Index returns an unsorted list of blocks that can be retrieved from
// this server.
-func (s *KeepService) Index(c *Client, prefix string) ([]KeepServiceIndexEntry, error) {
- return s.index(c, s.url("index/"+prefix))
+func (s *KeepService) Index(ctx context.Context, c *Client, prefix string) ([]KeepServiceIndexEntry, error) {
+ return s.index(ctx, c, s.url("index/"+prefix))
}
-func (s *KeepService) index(c *Client, url string) ([]KeepServiceIndexEntry, error) {
- req, err := http.NewRequest("GET", url, nil)
+func (s *KeepService) index(ctx context.Context, c *Client, url string) ([]KeepServiceIndexEntry, error) {
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
if err != nil {
- return nil, fmt.Errorf("NewRequest(%v): %v", url, err)
+ return nil, fmt.Errorf("NewRequestWithContext(%v): %v", url, err)
}
resp, err := c.Do(req)
if err != nil {
package arvados
import (
+ "context"
"net/http"
check "gopkg.in/check.v1"
APIHost: "zzzzz.arvadosapi.com",
AuthToken: "xyzzy",
}
- _, err := (&KeepService{}).IndexMount(client, "fake", "")
+ _, err := (&KeepService{}).IndexMount(context.Background(), client, "fake", "")
c.Check(err, check.ErrorMatches, `.*timeout.*`)
}
pg (1.1.4)
power_assert (1.1.4)
public_suffix (4.0.3)
- rack (2.2.2)
+ rack (2.2.3)
rack-test (0.6.3)
rack (>= 1.0)
rails (5.0.7.2)
logger.warn "User #{current_user.uuid} tried to set ownership of #{self.class.to_s} #{self.uuid} but does not have permission to write #{which} owner_uuid #{check_uuid}"
errors.add :owner_uuid, "cannot be set or changed without write permission on #{which} owner"
raise PermissionDeniedError
+ elsif rsc_class == Group && Group.find_by_uuid(owner_uuid).group_class != "project"
+ errors.add :owner_uuid, "must be a project"
+ raise PermissionDeniedError
end
end
else
# itself.
if !current_user.can?(write: self.uuid)
logger.warn "User #{current_user.uuid} tried to modify #{self.class.to_s} #{self.uuid} without write permission"
- errors.add :uuid, "is not writable"
+ errors.add :uuid, " #{uuid} is not writable by #{current_user.uuid}"
raise PermissionDeniedError
end
end
attribute :properties, :jsonbHash, default: {}
validate :ensure_filesystem_compatible_name
+ validate :check_group_class
before_create :assign_name
after_create :after_ownership_change
after_create :update_trash
before_update :before_ownership_change
after_update :after_ownership_change
+ after_create :add_role_manage_link
+
after_update :update_trash
before_destroy :clear_permissions_and_trash
super if group_class == 'project'
end
+ def check_group_class
+ if group_class != 'project' && group_class != 'role'
+ errors.add :group_class, "value must be one of 'project' or 'role', was '#{group_class}'"
+ end
+ if group_class_changed? && !group_class_was.nil?
+ errors.add :group_class, "cannot be modified after record is created"
+ end
+ end
+
def update_trash
if trash_at_changed? or owner_uuid_changed?
# The group was added or removed from the trash.
end
true
end
+
+ def ensure_owner_uuid_is_permitted
+ if group_class == "role"
+ @requested_manager_uuid = nil
+ if new_record?
+ @requested_manager_uuid = owner_uuid
+ self.owner_uuid = system_user_uuid
+ return true
+ end
+ if self.owner_uuid != system_user_uuid
+ raise "Owner uuid for role must be system user"
+ end
+ raise PermissionDeniedError unless current_user.can?(manage: uuid)
+ true
+ else
+ super
+ end
+ end
+
+ def add_role_manage_link
+ if group_class == "role" && @requested_manager_uuid
+ act_as_system_user do
+ Link.create!(tail_uuid: @requested_manager_uuid,
+ head_uuid: self.uuid,
+ link_class: "permission",
+ name: "can_manage")
+ end
+ end
+ end
end
attribute :properties, :jsonbHash, default: {}
validate :name_links_are_obsolete
- before_create :permission_to_attach_to_objects
- before_update :permission_to_attach_to_objects
+ validate :permission_to_attach_to_objects
+ before_update :restrict_alter_permissions
after_update :call_update_permissions
after_create :call_update_permissions
before_destroy :clear_permissions
# All users can write links that don't affect permissions
return true if self.link_class != 'permission'
+ if PERM_LEVEL[self.name].nil?
+ errors.add(:name, "is invalid permission, must be one of 'can_read', 'can_write', 'can_manage', 'can_login'")
+ return false
+ end
+
+ rsc_class = ArvadosModel::resource_class_for_uuid tail_uuid
+ if rsc_class == Group
+ tail_obj = Group.find_by_uuid(tail_uuid)
+ if tail_obj.nil?
+ errors.add(:tail_uuid, "does not exist")
+ return false
+ end
+ if tail_obj.group_class != "role"
+ errors.add(:tail_uuid, "must be a user or role, was group with group_class #{tail_obj.group_class}")
+ return false
+ end
+ elsif rsc_class != User
+ errors.add(:tail_uuid, "must be a user or role")
+ return false
+ end
+
# Administrators can grant permissions
return true if current_user.is_admin
head_obj = ArvadosModel.find_by_uuid(head_uuid)
# No permission links can be pointed to past collection versions
- return false if head_obj.is_a?(Collection) && head_obj.current_version_uuid != head_uuid
+ if head_obj.is_a?(Collection) && head_obj.current_version_uuid != head_uuid
+ errors.add(:head_uuid, "cannot point to a past version of a collection")
+ return false
+ end
# All users can grant permissions on objects they own or can manage
return true if current_user.can?(manage: head_obj)
false
end
+ def restrict_alter_permissions
+ return true if self.link_class != 'permission' && self.link_class_was != 'permission'
+
+ return true if current_user.andand.uuid == system_user.uuid
+
+ if link_class_changed? || tail_uuid_changed? || head_uuid_changed?
+ raise "Can only alter permission link level"
+ end
+ end
+
PERM_LEVEL = {
'can_read' => 1,
'can_login' => 1,
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'fix_roles_projects'
+
+class FixRolesProjects < ActiveRecord::Migration[5.0]
+ def up
+ # defined in a function for easy testing.
+ fix_roles_projects
+ end
+
+ def down
+ # This migration is not reversible. However, the results are
+ # backwards compatible.
+ end
+end
('20190808145904'),
('20190809135453'),
('20190905151603'),
-('20200501150153');
+('20200501150153'),
+('20200602141328');
ActiveRecord::Base.transaction do
Group.where(uuid: system_group_uuid).
first_or_create!(name: "System group",
- description: "System group") do |g|
+ description: "System group",
+ group_class: "role") do |g|
g.save!
User.all.collect(&:uuid).each do |user_uuid|
Link.create!(link_class: 'permission',
end
end
- def empty_collection_uuid
+ def empty_collection_pdh
'd41d8cd98f00b204e9800998ecf8427e+0'
end
act_as_system_user do
ActiveRecord::Base.transaction do
Collection.
- where(portable_data_hash: empty_collection_uuid).
- first_or_create!(manifest_text: '', owner_uuid: anonymous_group.uuid)
+ where(portable_data_hash: empty_collection_pdh).
+ first_or_create(manifest_text: '', owner_uuid: system_user.uuid, name: "empty collection") do |c|
+ c.save!
+ Link.where(tail_uuid: anonymous_group.uuid,
+ head_uuid: c.uuid,
+ link_class: 'permission',
+ name: 'can_read').
+ first_or_create!
+ c
+ end
end
end
end
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'update_permissions'
+
+include CurrentApiClient
+
+def fix_roles_projects
+ batch_update_permissions do
+ # This migration is not reversible. However, the behavior it
+ # enforces is backwards-compatible, and most of the time there
+ # shouldn't be anything to do at all.
+ act_as_system_user do
+ ActiveRecord::Base.transaction do
+ Group.where("group_class != 'project' or group_class is null").each do |g|
+ # 1) any group not group_class != project becomes a 'role' (both empty and invalid groups)
+ old_owner = g.owner_uuid
+ g.owner_uuid = system_user_uuid
+ g.group_class = 'role'
+ g.save_with_unique_name!
+
+ if old_owner != system_user_uuid
+ # 2) Ownership of a role becomes a can_manage link
+ Link.create!(link_class: 'permission',
+ name: 'can_manage',
+ tail_uuid: old_owner,
+ head_uuid: g.uuid)
+ end
+ end
+
+ ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
+ next if [ApiClientAuthorization,
+ AuthorizedKey,
+ Log,
+ Group].include?(klass)
+ next if !klass.columns.collect(&:name).include?('owner_uuid')
+
+ # 3) If a role owns anything, give it to system user and it
+ # becomes a can_manage link
+ klass.joins("join groups on groups.uuid=#{klass.table_name}.owner_uuid and groups.group_class='role'").each do |owned|
+ Link.create!(link_class: 'permission',
+ name: 'can_manage',
+ tail_uuid: owned.owner_uuid,
+ head_uuid: owned.uuid)
+ owned.owner_uuid = system_user_uuid
+ owned.save_with_unique_name!
+ end
+ end
+
+ Group.joins("join groups as g2 on g2.uuid=groups.owner_uuid and g2.group_class='role'").each do |owned|
+ Link.create!(link_class: 'permission',
+ name: 'can_manage',
+ tail_uuid: owned.owner_uuid,
+ head_uuid: owned.uuid)
+ owned.owner_uuid = system_user_uuid
+ owned.save_with_unique_name!
+ end
+
+ # 4) Projects can't have outgoing permission links. Just
+ # print a warning and delete them.
+ q = ActiveRecord::Base.connection.exec_query %{
+select links.uuid from links, groups where groups.uuid = links.tail_uuid and
+ links.link_class = 'permission' and groups.group_class = 'project'
+}
+ q.each do |lu|
+ ln = Link.find_by_uuid(lu['uuid'])
+ puts "WARNING: Projects cannot have outgoing permission links, removing '#{ln.name}' link #{ln.uuid} from #{ln.tail_uuid} to #{ln.head_uuid}"
+ Rails.logger.warn "Projects cannot have outgoing permission links, removing '#{ln.name}' link #{ln.uuid} from #{ln.tail_uuid} to #{ln.head_uuid}"
+ ln.destroy!
+ end
+ end
+ end
+ end
+end
CAN_MANAGE_PERM = 3
def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil
+ return if Thread.current[:suppress_update_permissions]
+
#
# Update a subset of the permission table affected by adding or
# removing a particular permission relationship (ownership or a
def check_permissions_against_full_refresh
# No-op except when running tests
- return unless Rails.env == 'test' and !Thread.current[:no_check_permissions_against_full_refresh]
+ return unless Rails.env == 'test' and !Thread.current[:no_check_permissions_against_full_refresh] and !Thread.current[:suppress_update_permissions]
# For checking correctness of the incremental permission updates.
# Check contents of the current 'materialized_permission' table
end
end
+def batch_update_permissions
+ check_perm_was = Thread.current[:suppress_update_permissions]
+ Thread.current[:suppress_update_permissions] = true
+ begin
+ yield
+ ensure
+ Thread.current[:suppress_update_permissions] = check_perm_was
+ refresh_permissions
+ end
+end
+
# Used to account for permissions that a user gains by having
# can_manage on another user.
#
empty:
uuid: zzzzz-4zz18-gs9ooj1h9sd5mde
current_version_uuid: zzzzz-4zz18-gs9ooj1h9sd5mde
- # Empty collection owned by anonymous_group is added with rake db:seed.
portable_data_hash: d41d8cd98f00b204e9800998ecf8427e+0
owner_uuid: zzzzz-tpzed-000000000000000
created_at: 2014-06-11T17:22:54Z
modified_at: 2014-06-11T17:22:54Z
updated_at: 2014-06-11T17:22:54Z
manifest_text: ""
- name: empty_collection
+ name: "empty collection for python test"
foo_collection_in_aproject:
uuid: zzzzz-4zz18-fy296fx3hot09f7
uuid: zzzzz-j7d0g-it30l961gq3t0oi
owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
name: Public
- description: Public Group
+ description: Public Project
+ group_class: project
+
+public_role:
+ uuid: zzzzz-j7d0g-jt30l961gq3t0oi
+ owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ name: Public Role
+ description: Public Role
group_class: role
private:
uuid: zzzzz-j7d0g-rew6elm53kancon
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
name: Private
- description: Private Group
+ description: Private Project
+ group_class: project
+
+private_role:
+ uuid: zzzzz-j7d0g-pew6elm53kancon
+ owner_uuid: zzzzz-tpzed-000000000000000
+ name: Private Role
+ description: Private Role
group_class: role
private_and_can_read_foofile:
uuid: zzzzz-j7d0g-22xp1wpjul508rk
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
name: Private and Can Read Foofile
description: Another Private Group
group_class: role
future_project_viewing_group:
uuid: zzzzz-j7d0g-futrprojviewgrp
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ owner_uuid: zzzzz-tpzed-000000000000000
created_at: 2014-04-21 15:37:48 -0400
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
modified_at: 2014-05-03 18:50:08 -0400
updated_at: 2014-05-03 18:50:08 -0400
name: Owned by bad group b
+ group_class: project
bad_group_has_ownership_cycle_b:
uuid: zzzzz-j7d0g-0077nzts8c178lw
modified_at: 2014-05-03 18:50:08 -0400
updated_at: 2014-05-03 18:50:08 -0400
name: Owned by bad group a
+ group_class: project
anonymous_group:
uuid: zzzzz-j7d0g-anonymouspublic
description: Test project belonging to FUSE test user
group_class: project
-group_with_no_class:
- uuid: zzzzz-j7d0g-groupwithnoclas
- owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- created_at: 2014-04-21 15:37:48 -0400
- modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
- modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
- modified_at: 2014-04-21 15:37:48 -0400
- updated_at: 2014-04-21 15:37:48 -0400
- name: group_with_no_class
- description: This group has no class at all. So rude!
-
# This wouldn't pass model validation, but it enables a workbench
# infinite-loop test. See #4389
project_owns_itself:
head_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
properties: {}
-foo_file_readable_by_aproject:
+foo_file_readable_by_project_viewer:
uuid: zzzzz-o0j2j-fp1d8395ldqw22p
owner_uuid: zzzzz-tpzed-000000000000000
created_at: 2014-01-24 20:42:26 -0800
modified_by_user_uuid: zzzzz-tpzed-000000000000000
modified_at: 2014-01-24 20:42:26 -0800
updated_at: 2014-01-24 20:42:26 -0800
- tail_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+ tail_uuid: zzzzz-tpzed-projectviewer1a
link_class: permission
name: can_read
head_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
name: can_read
head_uuid: zzzzz-4zz18-taggedcolletion
properties: {}
+
+active_manages_viewing_group:
+ uuid: zzzzz-o0j2j-activemanagesvi
+ owner_uuid: zzzzz-tpzed-000000000000000
+ created_at: 2014-01-24 20:42:26 -0800
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-000000000000000
+ modified_at: 2014-01-24 20:42:26 -0800
+ updated_at: 2014-01-24 20:42:26 -0800
+ tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ link_class: permission
+ name: can_manage
+ head_uuid: zzzzz-j7d0g-futrprojviewgrp
+ properties: {}
@controller = Arvados::V1::GroupsController.new
authorize_with :active
post :create, params: {
- group: {},
+ group: {group_class: "project"},
ensure_unique_name: boolparam
}
assert_response :success
post :create, params: {
group: {
name: groups(:aproject).name,
- owner_uuid: groups(:aproject).owner_uuid
+ owner_uuid: groups(:aproject).owner_uuid,
+ group_class: "project"
},
ensure_unique_name: boolparam
}
class Arvados::V1::FiltersTest < ActionController::TestCase
test '"not in" filter passes null values' do
- @controller = Arvados::V1::GroupsController.new
+ @controller = Arvados::V1::ContainerRequestsController.new
authorize_with :admin
get :index, params: {
- filters: [ ['group_class', 'not in', ['project']] ],
- controller: 'groups',
+ filters: [ ['container_uuid', 'not in', ['zzzzz-dz642-queuedcontainer', 'zzzzz-dz642-runningcontainr']] ],
+ controller: 'container_requests',
}
assert_response :success
found = assigns(:objects)
- assert_includes(found.collect(&:group_class), nil,
- "'group_class not in ['project']' filter should pass null")
+ assert_includes(found.collect(&:container_uuid), nil,
+ "'container_uuid not in [zzzzz-dz642-queuedcontainer, zzzzz-dz642-runningcontainr]' filter should pass null")
end
test 'error message for non-array element in filters array' do
end
assert_includes group_uuids, groups(:aproject).uuid
assert_includes group_uuids, groups(:asubproject).uuid
+ assert_includes group_uuids, groups(:private).uuid
assert_not_includes group_uuids, groups(:system_group).uuid
- assert_not_includes group_uuids, groups(:private).uuid
+ assert_not_includes group_uuids, groups(:private_and_can_read_foofile).uuid
end
test "get list of groups that are not projects" do
end
assert_not_includes group_uuids, groups(:aproject).uuid
assert_not_includes group_uuids, groups(:asubproject).uuid
- assert_includes group_uuids, groups(:private).uuid
- assert_includes group_uuids, groups(:group_with_no_class).uuid
end
test "get list of groups with bogus group_class" do
assert_equal 0, json_response['included'].length
end
- test 'get shared, owned by non-project' do
+ test 'get shared, add permission link' do
authorize_with :user_bar_in_sharing_group
act_as_system_user do
- Group.find_by_uuid(groups(:project_owned_by_foo).uuid).update!(owner_uuid: groups(:group_for_sharing_tests).uuid)
+ Link.create!(tail_uuid: groups(:group_for_sharing_tests).uuid,
+ head_uuid: groups(:project_owned_by_foo).uuid,
+ link_class: 'permission',
+ name: 'can_manage')
end
get :shared, params: {:filters => [["group_class", "=", "project"]], :include => "owner_uuid"}
assert_equal 1, json_response['items'].length
- assert_equal json_response['items'][0]["uuid"], groups(:project_owned_by_foo).uuid
+ assert_equal groups(:project_owned_by_foo).uuid, json_response['items'][0]["uuid"]
assert_equal 1, json_response['included'].length
- assert_equal json_response['included'][0]["uuid"], groups(:group_for_sharing_tests).uuid
+ assert_equal users(:user_foo_in_sharing_group).uuid, json_response['included'][0]["uuid"]
end
### contents with exclude_home_project
assert_equal 0, json_response['included'].length
end
- test 'contents, exclude home, owned by non-project' do
+ test 'contents, exclude home, add permission link' do
authorize_with :user_bar_in_sharing_group
act_as_system_user do
- Group.find_by_uuid(groups(:project_owned_by_foo).uuid).update!(owner_uuid: groups(:group_for_sharing_tests).uuid)
+ Link.create!(tail_uuid: groups(:group_for_sharing_tests).uuid,
+ head_uuid: groups(:project_owned_by_foo).uuid,
+ link_class: 'permission',
+ name: 'can_manage')
end
get :contents, params: {:include => "owner_uuid", :exclude_home_project => true}
assert_equal 1, json_response['items'].length
- assert_equal json_response['items'][0]["uuid"], groups(:project_owned_by_foo).uuid
+ assert_equal groups(:project_owned_by_foo).uuid, json_response['items'][0]["uuid"]
assert_equal 1, json_response['included'].length
- assert_equal json_response['included'][0]["uuid"], groups(:group_for_sharing_tests).uuid
+ assert_equal users(:user_foo_in_sharing_group).uuid, json_response['included'][0]["uuid"]
end
test 'contents, exclude home, with parent specified' do
test "get_all_permissions obeys group permissions" do
act_as_user system_user do
r = Repository.create!(name: 'admin/groupcanwrite', owner_uuid: users(:admin).uuid)
- g = Group.create!(group_class: 'group', name: 'repo-writers')
+ g = Group.create!(group_class: 'role', name: 'repo-writers')
u1 = users(:active)
u2 = users(:spectator)
Link.create!(tail_uuid: g.uuid, head_uuid: r.uuid, link_class: 'permission', name: 'can_manage')
Link.create!(tail_uuid: u2.uuid, head_uuid: g.uuid, link_class: 'permission', name: 'can_read')
r = Repository.create!(name: 'admin/groupreadonly', owner_uuid: users(:admin).uuid)
- g = Group.create!(group_class: 'group', name: 'repo-readers')
+ g = Group.create!(group_class: 'role', name: 'repo-readers')
u1 = users(:active)
u2 = users(:spectator)
Link.create!(tail_uuid: g.uuid, head_uuid: r.uuid, link_class: 'permission', name: 'can_read')
test "non-admin user gets only safe attributes from users#show" do
g = act_as_system_user do
- create :group
+ create :group, group_class: "role"
end
users = create_list :active_user, 2, join_groups: [g]
token = create :token, user: users[0]
[2, 4].each do |limit|
test "non-admin user can limit index to #{limit}" do
g = act_as_system_user do
- create :group
+ create :group, group_class: "role"
end
users = create_list :active_user, 4, join_groups: [g]
token = create :token, user: users[0]
post "/arvados/v1/groups",
params: {
group: {
- name: name
+ name: name,
+ group_class: "project"
},
async: true
},
class PermissionsTest < ActionDispatch::IntegrationTest
include DbCurrentTime
- include CurrentApiClient # for empty_collection
fixtures :users, :groups, :api_client_authorizations, :collections
test "adding and removing direct can_read links" do
tail_uuid: users(:spectator).uuid,
link_class: 'permission',
name: 'can_read',
- head_uuid: groups(:private).uuid,
+ head_uuid: groups(:private_role).uuid,
properties: {}
}
},
params: {
:format => :json,
:link => {
- tail_uuid: groups(:private).uuid,
+ tail_uuid: groups(:private_role).uuid,
link_class: 'permission',
name: 'can_read',
head_uuid: collections(:foo_file).uuid,
params: {
:format => :json,
:link => {
- tail_uuid: groups(:private).uuid,
+ tail_uuid: groups(:private_role).uuid,
link_class: 'permission',
name: 'can_read',
head_uuid: collections(:foo_file).uuid,
tail_uuid: users(:spectator).uuid,
link_class: 'permission',
name: 'can_read',
- head_uuid: groups(:private).uuid,
+ head_uuid: groups(:private_role).uuid,
properties: {}
}
},
tail_uuid: users(:spectator).uuid,
link_class: 'permission',
name: 'can_read',
- head_uuid: groups(:private).uuid,
+ head_uuid: groups(:private_role).uuid,
properties: {}
}
},
params: {
:format => :json,
:link => {
- tail_uuid: groups(:private).uuid,
+ tail_uuid: groups(:private_role).uuid,
link_class: 'permission',
name: 'can_read',
head_uuid: groups(:empty_lonely_group).uuid,
test "active user can read the empty collection" do
# The active user should be able to read the empty collection.
- get("/arvados/v1/collections/#{empty_collection_uuid}",
+ get("/arvados/v1/collections/#{empty_collection_pdh}",
params: {:format => :json},
headers: auth(:active))
assert_response :success
act_as_system_user do
puts("Time spent creating records:", Benchmark.measure do
ActiveRecord::Base.transaction do
- root = Group.create!(owner_uuid: users(:permission_perftest).uuid)
+ root = Group.create!(owner_uuid: users(:permission_perftest).uuid, group_class: "project")
n += 1
a = create_eight root.uuid
n += 8
while longstring.length < 2**16
longstring = longstring + longstring
end
- g = Group.create! name: 'Has a long description', description: longstring
+ g = Group.create! name: 'Has a long description', description: longstring, group_class: "project"
g = Group.find_by_uuid g.uuid
assert_equal g.description, longstring
end
test 'create and retrieve using created_at time' do
set_user_from_auth :active
- group = Group.create! name: 'test create and retrieve group'
+ group = Group.create! name: 'test create and retrieve group', group_class: "project"
assert group.valid?, "group is not valid"
results = Group.where(created_at: group.created_at)
test 'create and update twice and expect different update times' do
set_user_from_auth :active
- group = Group.create! name: 'test create and retrieve group'
+ group = Group.create! name: 'test create and retrieve group', group_class: "project"
assert group.valid?, "group is not valid"
# update 1
# SPDX-License-Identifier: AGPL-3.0
require 'test_helper'
+require 'fix_roles_projects'
class GroupTest < ActiveSupport::TestCase
test "cannot create a new ownership cycle" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
- g_bar = Group.create!(name: "bar")
+ g_foo = Group.create!(name: "foo", group_class: "project")
+ g_bar = Group.create!(name: "bar", group_class: "project")
g_foo.owner_uuid = g_bar.uuid
assert g_foo.save, lambda { g_foo.errors.messages }
test "cannot create a single-object ownership cycle" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
+ g_foo = Group.create!(name: "foo", group_class: "project")
assert g_foo.save
# Ensure I have permission to manage this group even when its owner changes
assert g_foo.errors.messages[:owner_uuid].join(" ").match(/ownership cycle/)
end
+ test "cannot create a group that is not a 'role' or 'project'" do
+ set_user_from_auth :active_trustedclient
+
+ assert_raises(ActiveRecord::RecordInvalid) do
+ Group.create!(name: "foo")
+ end
+
+ assert_raises(ActiveRecord::RecordInvalid) do
+ Group.create!(name: "foo", group_class: "")
+ end
+
+ assert_raises(ActiveRecord::RecordInvalid) do
+ Group.create!(name: "foo", group_class: "bogus")
+ end
+ end
+
+ test "cannot change group_class on an already created group" do
+ set_user_from_auth :active_trustedclient
+ g = Group.create!(name: "foo", group_class: "role")
+ assert_raises(ActiveRecord::RecordInvalid) do
+ g.update_attributes!(group_class: "project")
+ end
+ end
+
+ test "role cannot own things" do
+ set_user_from_auth :active_trustedclient
+ role = Group.create!(name: "foo", group_class: "role")
+ assert_raises(ArvadosModel::PermissionDeniedError) do
+ Collection.create!(name: "bzzz123", owner_uuid: role.uuid)
+ end
+
+ c = Collection.create!(name: "bzzz124")
+ assert_raises(ArvadosModel::PermissionDeniedError) do
+ c.update_attributes!(owner_uuid: role.uuid)
+ end
+ end
+
test "trash group hides contents" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
+ g_foo = Group.create!(name: "foo", group_class: "project")
col = Collection.create!(owner_uuid: g_foo.uuid)
assert Collection.readable_by(users(:active)).where(uuid: col.uuid).any?
test "trash group" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
- g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid)
- g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid)
+ g_foo = Group.create!(name: "foo", group_class: "project")
+ g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid, group_class: "project")
+ g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid, group_class: "project")
assert Group.readable_by(users(:active)).where(uuid: g_foo.uuid).any?
assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
test "trash subgroup" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
- g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid)
- g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid)
+ g_foo = Group.create!(name: "foo", group_class: "project")
+ g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid, group_class: "project")
+ g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid, group_class: "project")
assert Group.readable_by(users(:active)).where(uuid: g_foo.uuid).any?
assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
test "trash subsubgroup" do
set_user_from_auth :active_trustedclient
- g_foo = Group.create!(name: "foo")
- g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid)
- g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid)
+ g_foo = Group.create!(name: "foo", group_class: "project")
+ g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid, group_class: "project")
+ g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid, group_class: "project")
assert Group.readable_by(users(:active)).where(uuid: g_foo.uuid).any?
assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
test "trashed does not propagate across permission links" do
set_user_from_auth :admin
- g_foo = Group.create!(name: "foo")
+ g_foo = Group.create!(name: "foo", group_class: "role")
u_bar = User.create!(first_name: "bar")
assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).any?
set_user_from_auth :active
["", "{SOLIDUS}"].each do |subst|
Rails.configuration.Collections.ForwardSlashNameSubstitution = subst
- g = Group.create
+ proj = Group.create group_class: "project"
+ role = Group.create group_class: "role"
[[nil, true],
["", true],
[".", false],
["../..", subst != ""],
["/", subst != ""],
].each do |name, valid|
- g.name = name
- g.group_class = "role"
- assert_equal true, g.valid?
- g.group_class = "project"
- assert_equal valid, g.valid?, "#{name.inspect} should be #{valid ? "valid" : "invalid"}"
+ role.name = name
+ assert_equal true, role.valid?
+ proj.name = name
+ assert_equal valid, proj.valid?, "#{name.inspect} should be #{valid ? "valid" : "invalid"}"
end
end
end
+
+ def insert_group uuid, owner_uuid, name, group_class
+ q = ActiveRecord::Base.connection.exec_query %{
+insert into groups (uuid, owner_uuid, name, group_class, created_at, updated_at)
+ values ('#{uuid}', '#{owner_uuid}',
+ '#{name}', #{if group_class then "'"+group_class+"'" else 'NULL' end},
+ statement_timestamp(), statement_timestamp())
+}
+ uuid
+ end
+
+ test "migration to fix roles and projects" do
+ g1 = insert_group Group.generate_uuid, system_user_uuid, 'group with no class', nil
+ g2 = insert_group Group.generate_uuid, users(:active).uuid, 'role owned by a user', 'role'
+
+ g3 = insert_group Group.generate_uuid, system_user_uuid, 'role that owns a project', 'role'
+ g4 = insert_group Group.generate_uuid, g3, 'the project', 'project'
+
+ g5 = insert_group Group.generate_uuid, users(:active).uuid, 'a project with an outgoing permission link', 'project'
+
+ g6 = insert_group Group.generate_uuid, system_user_uuid, 'name collision', 'role'
+ g7 = insert_group Group.generate_uuid, users(:active).uuid, 'name collision', 'role'
+
+ refresh_permissions
+
+ act_as_system_user do
+ l1 = Link.create!(link_class: 'permission', name: 'can_manage', tail_uuid: g3, head_uuid: g4)
+ q = ActiveRecord::Base.connection.exec_query %{
+update links set tail_uuid='#{g5}' where uuid='#{l1.uuid}'
+}
+ refresh_permissions
+ end
+
+ assert_equal nil, Group.find_by_uuid(g1).group_class
+ assert_equal users(:active).uuid, Group.find_by_uuid(g2).owner_uuid
+ assert_equal g3, Group.find_by_uuid(g4).owner_uuid
+ assert !Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?
+ assert !Link.where(tail_uuid: g3, head_uuid: g4, link_class: "permission", name: "can_manage").any?
+ assert Link.where(link_class: 'permission', name: 'can_manage', tail_uuid: g5, head_uuid: g4).any?
+
+ fix_roles_projects
+
+ assert_equal 'role', Group.find_by_uuid(g1).group_class
+ assert_equal system_user_uuid, Group.find_by_uuid(g2).owner_uuid
+ assert_equal system_user_uuid, Group.find_by_uuid(g4).owner_uuid
+ assert Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?
+ assert Link.where(tail_uuid: g3, head_uuid: g4, link_class: "permission", name: "can_manage").any?
+ assert !Link.where(link_class: 'permission', name: 'can_manage', tail_uuid: g5, head_uuid: g4).any?
+ end
end
Group.all
[User, Group].each do |o_class|
test "create object with legit #{o_class} owner" do
- o = o_class.create!
+ if o_class == Group
+ o = o_class.create! group_class: "project"
+ else
+ o = o_class.create!
+ end
i = Specimen.create(owner_uuid: o.uuid)
assert i.valid?, "new item should pass validation"
assert i.uuid, "new item should have an ID"
[User, Group].each do |new_o_class|
test "change owner from legit #{o_class} to legit #{new_o_class} owner" do
- o = o_class.create!
+ o = if o_class == Group
+ o_class.create! group_class: "project"
+ else
+ o_class.create!
+ end
i = Specimen.create!(owner_uuid: o.uuid)
- new_o = new_o_class.create!
+
+ new_o = if new_o_class == Group
+ new_o_class.create! group_class: "project"
+ else
+ new_o_class.create!
+ end
+
assert(Specimen.where(uuid: i.uuid).any?,
"new item should really be in DB")
assert(i.update_attributes(owner_uuid: new_o.uuid),
end
test "delete #{o_class} that owns nothing" do
- o = o_class.create!
+ if o_class == Group
+ o = o_class.create! group_class: "project"
+ else
+ o = o_class.create!
+ end
assert(o_class.where(uuid: o.uuid).any?,
"new #{o_class} should really be in DB")
assert(o.destroy, "should delete #{o_class} that owns nothing")
test "change uuid of #{o_class} that owns nothing" do
# (we're relying on our admin credentials here)
- o = o_class.create!
+ if o_class == Group
+ o = o_class.create! group_class: "project"
+ else
+ o = o_class.create!
+ end
assert(o_class.where(uuid: o.uuid).any?,
"new #{o_class} should really be in DB")
old_uuid = o.uuid
end
test "readable_by" do
- set_user_from_auth :active_trustedclient
+ set_user_from_auth :admin
ob = Collection.create!
Link.create!(tail_uuid: users(:active).uuid,
end
test "writable_by" do
- set_user_from_auth :active_trustedclient
+ set_user_from_auth :admin
ob = Collection.create!
Link.create!(tail_uuid: users(:active).uuid,
set_user_from_auth :admin
owner_grp = Group.create!(owner_uuid: users(:active).uuid, group_class: "role")
+
sp_grp = Group.create!(group_class: "project")
Link.create!(link_class: 'permission',
# anyone any additional permissions.)
g = nil
act_as_user manager do
- g = create :group, name: "NoBigSecret Lab"
+ g = create :group, name: "NoBigSecret Lab", group_class: "role"
assert_empty(User.readable_by(manager).where(uuid: minion.uuid),
"saw a user I shouldn't see")
assert_raises(ArvadosModel::PermissionDeniedError,
"#{a.first_name} should not be able to see 'b' in the user list")
act_as_system_user do
- g = create :group
+ g = create :group, group_class: "role"
[a,b].each do |u|
create(:permission_link,
name: 'can_read', tail_uuid: u.uuid, head_uuid: g.uuid)
test "add user to group, then remove them" do
set_user_from_auth :admin
grp = Group.create!(owner_uuid: system_user_uuid, group_class: "role")
- col = Collection.create!(owner_uuid: grp.uuid)
+ col = Collection.create!(owner_uuid: system_user_uuid)
+
+ l0 = Link.create!(tail_uuid: grp.uuid,
+ head_uuid: col.uuid,
+ link_class: 'permission',
+ name: 'can_read')
+
assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
test "add user to group, then change permission level" do
set_user_from_auth :admin
- grp = Group.create!(owner_uuid: system_user_uuid, group_class: "role")
+ grp = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
col = Collection.create!(owner_uuid: grp.uuid)
assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
head_uuid: grp.uuid,
link_class: 'permission',
name: 'can_manage')
- l2 = Link.create!(tail_uuid: grp.uuid,
- head_uuid: users(:active).uuid,
- link_class: 'permission',
- name: 'can_read')
assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
assert users(:active).can?(read: col.uuid)
test "add user to group, then add overlapping permission link to group" do
set_user_from_auth :admin
- grp = Group.create!(owner_uuid: system_user_uuid, group_class: "role")
+ grp = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
col = Collection.create!(owner_uuid: grp.uuid)
assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
head_uuid: grp.uuid,
link_class: 'permission',
name: 'can_manage')
- l2 = Link.create!(tail_uuid: grp.uuid,
- head_uuid: users(:active).uuid,
- link_class: 'permission',
- name: 'can_read')
assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
assert users(:active).can?(read: col.uuid)
test "add user to group, then add overlapping permission link to subproject" do
set_user_from_auth :admin
- grp = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
- prj = Group.create!(owner_uuid: grp.uuid, group_class: "project")
+ grp = Group.create!(owner_uuid: system_user_uuid, group_class: "role")
+ prj = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
+
+ l0 = Link.create!(tail_uuid: grp.uuid,
+ head_uuid: prj.uuid,
+ link_class: 'permission',
+ name: 'can_manage')
+
assert_empty Group.readable_by(users(:active)).where(uuid: prj.uuid)
assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
super(FuseMagicTest, self).setUp(api=api)
self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
- self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
+ self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
cw = arvados.CollectionWriter()
import (
"bytes"
+ "context"
"crypto/md5"
"fmt"
"io"
defer bal.time("sweep", "wall clock time to run one full sweep")()
+ ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(cluster.Collections.BalanceTimeout.Duration()))
+ defer cancel()
+
var lbFile *os.File
if bal.LostBlocksFile != "" {
tmpfn := bal.LostBlocksFile + ".tmp"
if err = bal.CheckSanityEarly(client); err != nil {
return
}
+
+ // On a big site, indexing and sending trash/pull lists can
+ // take much longer than the usual 5 minute client
+ // timeout. From here on, we rely on the context deadline
+ // instead, aborting the entire operation if any part takes
+ // too long.
+ client.Timeout = 0
+
rs := bal.rendezvousState()
if runOptions.CommitTrash && rs != runOptions.SafeRendezvousState {
if runOptions.SafeRendezvousState != "" {
bal.logf("notice: KeepServices list has changed since last run")
}
bal.logf("clearing existing trash lists, in case the new rendezvous order differs from previous run")
- if err = bal.ClearTrashLists(client); err != nil {
+ if err = bal.ClearTrashLists(ctx, client); err != nil {
return
}
// The current rendezvous state becomes "safe" (i.e.,
// succeed in clearing existing trash lists.
nextRunOptions.SafeRendezvousState = rs
}
- if err = bal.GetCurrentState(client, cluster.Collections.BalanceCollectionBatch, cluster.Collections.BalanceCollectionBuffers); err != nil {
+
+ if err = bal.GetCurrentState(ctx, client, cluster.Collections.BalanceCollectionBatch, cluster.Collections.BalanceCollectionBuffers); err != nil {
return
}
bal.ComputeChangeSets()
lbFile = nil
}
if runOptions.CommitPulls {
- err = bal.CommitPulls(client)
+ err = bal.CommitPulls(ctx, client)
if err != nil {
// Skip trash if we can't pull. (Too cautious?)
return
}
}
if runOptions.CommitTrash {
- err = bal.CommitTrash(client)
+ err = bal.CommitTrash(ctx, client)
}
return
}
// We avoid this problem if we clear all trash lists before getting
// indexes. (We also assume there is only one rebalancing process
// running at a time.)
-func (bal *Balancer) ClearTrashLists(c *arvados.Client) error {
+func (bal *Balancer) ClearTrashLists(ctx context.Context, c *arvados.Client) error {
for _, srv := range bal.KeepServices {
srv.ChangeSet = &ChangeSet{}
}
- return bal.CommitTrash(c)
+ return bal.CommitTrash(ctx, c)
}
// GetCurrentState determines the current replication state, and the
// collection manifests in the database (API server).
//
// It encodes the resulting information in BlockStateMap.
-func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) error {
+func (bal *Balancer) GetCurrentState(ctx context.Context, c *arvados.Client, pageSize, bufs int) error {
+ ctx, cancel := context.WithCancel(ctx)
+ defer cancel()
+
defer bal.time("get_state", "wall clock time to get current state")()
bal.BlockStateMap = NewBlockStateMap()
go func(mounts []*KeepMount) {
defer wg.Done()
bal.logf("mount %s: retrieve index from %s", mounts[0], mounts[0].KeepService)
- idx, err := mounts[0].KeepService.IndexMount(c, mounts[0].UUID, "")
+ idx, err := mounts[0].KeepService.IndexMount(ctx, c, mounts[0].UUID, "")
if err != nil {
select {
case errs <- fmt.Errorf("%s: retrieve index: %v", mounts[0], err):
default:
}
+ cancel()
return
}
if len(errs) > 0 {
}
for range collQ {
}
+ cancel()
return
}
bal.collScanned++
wg.Add(1)
go func() {
defer wg.Done()
- err = EachCollection(c, pageSize,
+ err = EachCollection(ctx, c, pageSize,
func(coll arvados.Collection) error {
collQ <- coll
if len(errs) > 0 {
case errs <- err:
default:
}
+ cancel()
}
}()
// keepstore servers. This has the effect of increasing replication of
// existing blocks that are either underreplicated or poorly
// distributed according to rendezvous hashing.
-func (bal *Balancer) CommitPulls(c *arvados.Client) error {
+func (bal *Balancer) CommitPulls(ctx context.Context, c *arvados.Client) error {
defer bal.time("send_pull_lists", "wall clock time to send pull lists")()
return bal.commitAsync(c, "send pull list",
func(srv *KeepService) error {
- return srv.CommitPulls(c)
+ return srv.CommitPulls(ctx, c)
})
}
// CommitTrash sends the computed lists of trash requests to the
// keepstore servers. This has the effect of deleting blocks that are
// overreplicated or unreferenced.
-func (bal *Balancer) CommitTrash(c *arvados.Client) error {
+func (bal *Balancer) CommitTrash(ctx context.Context, c *arvados.Client) error {
defer bal.time("send_trash_lists", "wall clock time to send trash lists")()
return bal.commitAsync(c, "send trash list",
func(srv *KeepService) error {
- return srv.CommitTrash(c)
+ return srv.CommitTrash(ctx, c)
})
}
package main
import (
+ "context"
"fmt"
"time"
//
// If pageSize > 0 it is used as the maximum page size in each API
// call; otherwise the maximum allowed page size is requested.
-func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection) error, progress func(done, total int)) error {
+func EachCollection(ctx context.Context, c *arvados.Client, pageSize int, f func(arvados.Collection) error, progress func(done, total int)) error {
if progress == nil {
progress = func(_, _ int) {}
}
for {
progress(callCount, expectCount)
var page arvados.CollectionList
- err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
+ err := c.RequestAndDecodeContext(ctx, &page, "GET", "arvados/v1/collections", nil, params)
if err != nil {
return err
}
package main
import (
+ "context"
"sync"
"time"
longestStreak := 0
var lastMod time.Time
sawUUID := make(map[string]bool)
- err := EachCollection(s.client, pageSize, func(c arvados.Collection) error {
+ err := EachCollection(context.Background(), s.client, pageSize, func(c arvados.Collection) error {
if c.ModifiedAt.IsZero() {
return nil
}
package main
import (
+ "context"
"encoding/json"
"fmt"
"io"
// CommitPulls sends the current list of pull requests to the storage
// server (even if the list is empty).
-func (srv *KeepService) CommitPulls(c *arvados.Client) error {
- return srv.put(c, "pull", srv.ChangeSet.Pulls)
+func (srv *KeepService) CommitPulls(ctx context.Context, c *arvados.Client) error {
+ return srv.put(ctx, c, "pull", srv.ChangeSet.Pulls)
}
// CommitTrash sends the current list of trash requests to the storage
// server (even if the list is empty).
-func (srv *KeepService) CommitTrash(c *arvados.Client) error {
- return srv.put(c, "trash", srv.ChangeSet.Trashes)
+func (srv *KeepService) CommitTrash(ctx context.Context, c *arvados.Client) error {
+ return srv.put(ctx, c, "trash", srv.ChangeSet.Trashes)
}
// Perform a PUT request at path, with data (as JSON) in the request
// body.
-func (srv *KeepService) put(c *arvados.Client, path string, data interface{}) error {
+func (srv *KeepService) put(ctx context.Context, c *arvados.Client, path string, data interface{}) error {
// We'll start a goroutine to do the JSON encoding, so we can
// stream it to the http client through a Pipe, rather than
// keeping the entire encoded version in memory.
}()
url := srv.URLBase() + "/" + path
- req, err := http.NewRequest("PUT", url, ioutil.NopCloser(jsonR))
+ req, err := http.NewRequestWithContext(ctx, "PUT", url, ioutil.NopCloser(jsonR))
if err != nil {
return fmt.Errorf("building request for %s: %v", url, err)
}
"log"
"net/http"
"os"
+ "os/signal"
+ "syscall"
"time"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
ServiceURL = flag.String("url", "", "specify scheme://host of a single keep service to exercise (instead of using all advertised services like normal clients)")
ServiceUUID = flag.String("uuid", "", "specify UUID of a single advertised keep service to exercise")
getVersion = flag.Bool("version", false, "Print version information and exit.")
+ RunTime = flag.Duration("run-time", 0, "time to run (e.g. 60s), or 0 to run indefinitely (default)")
)
func main() {
os.Exit(0)
}
- log.Printf("keep-exercise %s started", version)
+ stderr := log.New(os.Stderr, "", log.LstdFlags)
arv, err := arvadosclient.MakeArvadosClient()
if err != nil {
- log.Fatal(err)
+ stderr.Fatal(err)
}
kc, err := keepclient.MakeKeepClient(arv)
if err != nil {
- log.Fatal(err)
+ stderr.Fatal(err)
}
kc.Want_replicas = *Replicas
Transport: &transport,
}
- overrideServices(kc)
+ overrideServices(kc, stderr)
nextLocator := make(chan string, *ReadThreads+*WriteThreads)
- go countBeans(nextLocator)
+ go countBeans(nextLocator, stderr)
for i := 0; i < *WriteThreads; i++ {
nextBuf := make(chan []byte, 1)
- go makeBufs(nextBuf, i)
- go doWrites(kc, nextBuf, nextLocator)
+ go makeBufs(nextBuf, i, stderr)
+ go doWrites(kc, nextBuf, nextLocator, stderr)
}
for i := 0; i < *ReadThreads; i++ {
- go doReads(kc, nextLocator)
+ go doReads(kc, nextLocator, stderr)
}
<-make(chan struct{})
}
// Send struct{}{} to errorsChan when an error happens.
var errorsChan = make(chan struct{})
-func countBeans(nextLocator chan string) {
+func countBeans(nextLocator chan string, stderr *log.Logger) {
t0 := time.Now()
var tickChan <-chan time.Time
+ var endChan <-chan time.Time
+ c := make(chan os.Signal)
+ signal.Notify(c, os.Interrupt, syscall.SIGTERM)
if *StatsInterval > 0 {
tickChan = time.NewTicker(*StatsInterval).C
}
+ if *RunTime > 0 {
+ endChan = time.NewTicker(*RunTime).C
+ }
var bytesIn uint64
var bytesOut uint64
var errors uint64
+ var rateIn, rateOut float64
+ var maxRateIn, maxRateOut float64
+ var abort, printCsv bool
+ csv := log.New(os.Stdout, "", 0)
+ csv.Println("Timestamp,Elapsed,Read (bytes),Avg Read Speed (MiB/s),Peak Read Speed (MiB/s),Written (bytes),Avg Write Speed (MiB/s),Peak Write Speed (MiB/s),Errors,ReadThreads,WriteThreads,VaryRequest,VaryThread,BlockSize,Replicas,StatsInterval,ServiceURL,ServiceUUID,RunTime")
for {
select {
case <-tickChan:
- elapsed := time.Since(t0)
- log.Printf("%v elapsed: read %v bytes (%.1f MiB/s), wrote %v bytes (%.1f MiB/s), errors %d",
- elapsed,
- bytesIn, (float64(bytesIn) / elapsed.Seconds() / 1048576),
- bytesOut, (float64(bytesOut) / elapsed.Seconds() / 1048576),
- errors,
- )
+ printCsv = true
+ case <-endChan:
+ printCsv = true
+ abort = true
+ case <-c:
+ printCsv = true
+ abort = true
+ fmt.Print("\r") // Suppress the ^C print
case i := <-bytesInChan:
bytesIn += i
case o := <-bytesOutChan:
case <-errorsChan:
errors++
}
+ if printCsv {
+ elapsed := time.Since(t0)
+ rateIn = float64(bytesIn) / elapsed.Seconds() / 1048576
+ if rateIn > maxRateIn {
+ maxRateIn = rateIn
+ }
+ rateOut = float64(bytesOut) / elapsed.Seconds() / 1048576
+ if rateOut > maxRateOut {
+ maxRateOut = rateOut
+ }
+ csv.Printf("%v,%v,%v,%.1f,%.1f,%v,%.1f,%.1f,%d,%d,%d,%t,%t,%d,%d,%s,%s,%s,%s",
+ time.Now().Format("2006-01-02 15:04:05"),
+ elapsed,
+ bytesIn, rateIn, maxRateIn,
+ bytesOut, rateOut, maxRateOut,
+ errors,
+ *ReadThreads,
+ *WriteThreads,
+ *VaryRequest,
+ *VaryThread,
+ *BlockSize,
+ *Replicas,
+ *StatsInterval,
+ *ServiceURL,
+ *ServiceUUID,
+ *RunTime,
+ )
+ printCsv = false
+ }
+ if abort {
+ os.Exit(0)
+ }
}
}
-func makeBufs(nextBuf chan<- []byte, threadID int) {
+func makeBufs(nextBuf chan<- []byte, threadID int, stderr *log.Logger) {
buf := make([]byte, *BlockSize)
if *VaryThread {
binary.PutVarint(buf, int64(threadID))
if *VaryRequest {
rnd := make([]byte, randSize)
if _, err := io.ReadFull(rand.Reader, rnd); err != nil {
- log.Fatal(err)
+ stderr.Fatal(err)
}
buf = append(rnd, buf[randSize:]...)
}
}
}
-func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan<- string) {
+func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan<- string, stderr *log.Logger) {
for buf := range nextBuf {
locator, _, err := kc.PutB(buf)
if err != nil {
- log.Print(err)
+ stderr.Print(err)
errorsChan <- struct{}{}
continue
}
}
}
-func doReads(kc *keepclient.KeepClient, nextLocator <-chan string) {
+func doReads(kc *keepclient.KeepClient, nextLocator <-chan string, stderr *log.Logger) {
for locator := range nextLocator {
rdr, size, url, err := kc.Get(locator)
if err != nil {
- log.Print(err)
+ stderr.Print(err)
errorsChan <- struct{}{}
continue
}
n, err := io.Copy(ioutil.Discard, rdr)
rdr.Close()
if n != size || err != nil {
- log.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
+ stderr.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
errorsChan <- struct{}{}
continue
// Note we don't count the bytes received in
}
}
-func overrideServices(kc *keepclient.KeepClient) {
+func overrideServices(kc *keepclient.KeepClient, stderr *log.Logger) {
roots := make(map[string]string)
if *ServiceURL != "" {
roots["zzzzz-bi6l4-000000000000000"] = *ServiceURL
}
}
if len(roots) == 0 {
- log.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
+ stderr.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
}
} else {
return
log.Println("Default parent group not found, creating...")
}
groupData := map[string]string{
- "name": cfg.ParentGroupName,
- "owner_uuid": cfg.SysUserUUID,
+ "name": cfg.ParentGroupName,
+ "owner_uuid": cfg.SysUserUUID,
+ "group_class": "role",
}
if err := CreateGroup(cfg, &parentGroup, groupData); err != nil {
return fmt.Errorf("error creating system user owned group named %q: %s", groupData["name"], err)
params := arvados.ResourceListParams{
Filters: []arvados.Filter{{
- Attr: "owner_uuid",
+ Attr: "tail_uuid",
Operator: "=",
Operand: cfg.ParentGroupUUID,
}},
}
- results, err := GetAll(cfg.Client, "groups", params, &GroupList{})
+ results, err := GetAll(cfg.Client, "links", params, &LinkList{})
if err != nil {
return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote groups: %s", err)
}
for _, item := range results {
- group := item.(arvados.Group)
+ var group arvados.Group
+ err = GetGroup(cfg, &group, item.(arvados.Link).HeadUUID)
+ if err != nil {
+ return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote group: %s", err)
+ }
// Group -> User filter
g2uFilter := arvados.ResourceListParams{
Filters: []arvados.Filter{{
}, {
Attr: "owner_uuid",
Operator: "=",
- Operand: cfg.ParentGroupUUID,
+ Operand: cfg.SysUserUUID,
}, {
Attr: "group_class",
Operator: "=",
projects / arvados.git / commitdiff