# star / unstar the current project
def star
- links = Link.where(tail_uuid: current_user.uuid,
+ links = Link.where(owner_uuid: current_user.uuid,
head_uuid: @object.uuid,
link_class: 'star')
helper_method :my_starred_projects
def my_starred_projects user
return if defined?(@starred_projects) && @starred_projects
- links = Link.filter([['tail_uuid', '=', user.uuid],
+ links = Link.filter([['owner_uuid', 'in', ["#{Rails.configuration.ClusterID}-j7d0g-fffffffffffffff", user.uuid]],
['link_class', '=', 'star'],
['head_uuid', 'is_a', 'arvados#group']]).with_count("none").select(%w(head_uuid))
uuids = links.collect { |x| x.head_uuid }
case "$TARGET" in
centos*)
- fpm_depends+=(git bison make automake gcc gcc-c++)
+ fpm_depends+=(git bison make automake gcc gcc-c++ graphviz)
;;
debian* | ubuntu*)
- fpm_depends+=(git g++ bison zlib1g-dev make)
+ fpm_depends+=(git g++ bison zlib1g-dev make graphviz)
;;
esac
export RAILS_ENV=production
if ! $COMMAND_PREFIX bundle --version >/dev/null; then
- run_and_report "Installing bundle" $COMMAND_PREFIX gem install bundle
+ run_and_report "Installing bundler" $COMMAND_PREFIX gem install bundler --version 1.17.3
fi
run_and_report "Running bundle install" \
# set to --no-cache-dir to disable pip caching
CACHE_FLAG=
-MAINTAINER="Ward Vandewege <wvandewege@veritasgenetics.com>"
-VENDOR="Veritas Genetics, Inc."
+MAINTAINER="Arvados Package Maintainers <packaging@arvados.org>"
+VENDOR="The Arvados Project"
# End of user configuration
Type=notify
EnvironmentFile=-/etc/arvados/environment
ExecStart=/usr/bin/arvados-controller
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
Type=notify
EnvironmentFile=-/etc/arvados/environment
ExecStart=/usr/bin/arvados-dispatch-cloud
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
- admin/metrics.html.textile.liquid
- admin/health-checks.html.textile.liquid
- admin/management-token.html.textile.liquid
- - Cloud:
- - admin/storage-classes.html.textile.liquid
- - admin/spot-instances.html.textile.liquid
- - admin/cloudtest.html.textile.liquid
- Data Management:
- admin/collection-versioning.html.textile.liquid
- admin/collection-managed-properties.html.textile.liquid
- admin/keep-balance.html.textile.liquid
- admin/controlling-container-reuse.html.textile.liquid
- admin/logs-table-management.html.textile.liquid
+ - admin/workbench2-vocabulary.html.textile.liquid
+ - admin/storage-classes.html.textile.liquid
+ - Cloud:
+ - admin/spot-instances.html.textile.liquid
+ - admin/cloudtest.html.textile.liquid
- Other:
- install/migrate-docker19.html.textile.liquid
- admin/upgrade-crunch2.html.textile.liquid
- - admin/workbench2-vocabulary.html.textile.liquid
installguide:
- Overview:
- install/index.html.textile.liquid
This page describes how to enable and configure the collection versioning feature on the API server.
-h3. API Server configuration
+h3. Configuration
-There are 2 configuration settings that control this feature, both go on the @application.yml@ file.
+There are 2 configuration settings in the @Collections@ section of @config.yml@ that control this feature.
-h4. Setting: @collection_versioning@ (Boolean. Default: false)
+<pre>
+ Collections:
+ # If true, enable collection versioning.
+ # When a collection's preserve_version field is true or the current version
+ # is older than the amount of seconds defined on PreserveVersionIfIdle,
+ # a snapshot of the collection's previous state is created and linked to
+ # the current collection.
+ CollectionVersioning: false
-If @true@, collection versioning is enabled, meaning that new version records can be created. Note that if you set @collection_versioning@ to @false@ after being enabled, old versions will still be accessible, but further changes will not be versioned.
+ # This setting control the auto-save aspect of collection versioning, and can be set to:
+ # 0s = auto-create a new version on every update.
+ # -1s = never auto-create new versions.
+ # > 0s = auto-create a new version when older than the specified number of seconds.
+ PreserveVersionIfIdle: -1s
+</pre>
-h4. Setting: @preserve_version_if_idle@ (Numeric. Default: -1)
-
-This setting control the auto-save aspect of collection versioning, and can be set to:
-* @-1@: Never auto-save versions. Only save versions when the client ask for it by setting @preserve_version@ to @true@ on any given collection.
-* @0@: Preserve all versions every time a collection gets a versionable update.
-* @N@ (being N > 0): Preserve version when a collection gets a versionable update after a period of at least N seconds since the last time it was modified.
+Note that if you set @collection_versioning@ to @false@ after being enabled, old versions will still be accessible, but further changes will not be versioned.
h3. Using collection versioning
-"Discussed in the user guide":{{site.baseurl}}/user/topics/collection-versioning.html
\ No newline at end of file
+"Discussed in the user guide":{{site.baseurl}}/user/topics/collection-versioning.html
---
layout: default
navsection: installguide
-title: Migrating Configuration from v1.4 to v1.5
+title: Migrating Configuration from v1.4 to v2.0
...
{% comment %}
{% endcomment %}
{% include 'notebox_begin_warning' %}
-_New installations of Arvados 1.5+ can skip this section_
+_New installations of Arvados 2.0+ can skip this section_
{% include 'notebox_end' %}
-Arvados 1.5 migrates to a centralized configuration file for all components. The centralized Arvados configuration is @/etc/arvados/config.yml@. Components that support the new centralized configuration are listed below. During the migration period, legacy configuration files are still loaded and take precedence over the centralized configuration file.
+Arvados 2.0 migrates to a centralized configuration file for all components. The centralized Arvados configuration is @/etc/arvados/config.yml@. Components that support the new centralized configuration are listed below. During the migration period, legacy configuration files are still loaded and take precedence over the centralized configuration file.
h2. API server
---
layout: default
navsection: admin
-title: Controlling container reuse
+title: Preventing container reuse
...
{% comment %}
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-This page describes how an admin can control container reuse using the @arv@ command. This can be utilized to avoid reusing a completed container without disabling reuse for the corresponding steps in affected workflows. For example, if a container exited successfully but produced bad output, it may not be feasible to update the workflow immediately. Meanwhile, changing the state of the container from @Complete@ to @Cancelled@ will prevent it from being used in subsequent workflows.
+Sometimes a container exited successfully but produced bad output, and re-running the workflow will cause it to re-use the bad container instead of running a new container. One way to deal with this is to re-run the entire workflow with reuse disable. Another way is for the workflow author to tweak the input data or workflow so that on re-run it produces a distinct container request. However, for large or complex workflows both these options may be impractical.
-If a container is in the @Complete@ state, the following @arv@ command will change its state to @Cancelled@, where @xxxxx-xxxxx-xxxxxxxxxxxxxxx@ is the @UUID@ of the container:
+To prevent an individual container from being reused in later workflows, an admin can manually change the state of the bad container record from @Complete@ to @Cancelled@. The following @arv@ command demonstrates how change a container state to @Cancelled@, where @xxxxx-xxxxx-xxxxxxxxxxxxxxx@ is the @UUID@ of the container:
<pre>arv container update -u xxxxx-xxxxx-xxxxxxxxxxxxxxx -c '{"state":"Cancelled"}'</pre>
-
-Use the following command to list all containers that exited with 0 and were then cancelled:
-
-<pre>arv container list --filters='[["state", "=", "Cancelled"], ["exit_code", "=", 0]]'</pre>See the "arv CLI tool overview":{{site.baseurl}}/sdk/cli/index.html for more details about using the @arv@ command.
As a result, this table grows indefinitely, even on sites where policy does not require an audit log; making backups, migrations, and upgrades unnecessarily slow and painful.
-h3. API Server configuration
+h3. Configuration
-To solve the problem mentioned above, the API server offers the possibility to limit the amount of log information stored on the table:
+To solve the problem mentioned above, the @AuditLogs@ section of @config.yml@ offers several options to limit the amount of log information stored on the table:
<pre>
-# Attributes to suppress in events and audit logs. Notably,
-# specifying ["manifest_text"] here typically makes the database
-# smaller and faster.
-#
-# Warning: Using any non-empty value here can have undesirable side
-# effects for any client or component that relies on event logs.
-# Use at your own risk.
-unlogged_attributes: []
+ AuditLogs:
+ # Time to keep audit logs. (An audit log is a row added
+ # to the "logs" table in the PostgreSQL database each time an
+ # Arvados object is created, modified, or deleted.)
+ #
+ # Currently, websocket event notifications rely on audit logs, so
+ # this should not be set lower than 5 minutes.
+ MaxAge: 336h
+
+ # Maximum number of log rows to delete in a single SQL transaction,
+ # to prevent surprises and avoid bad database behavior
+ # (especially the first time the cleanup job runs on an existing
+ # cluster with a huge backlog) a maximum number of rows to
+ # delete in a single transaction.
+ #
+ # If MaxDeleteBatch is 0, log entries will never be
+ # deleted by Arvados. Cleanup can be done by an external process
+ # without affecting any Arvados system processes, as long as very
+ # recent (<5 minutes old) logs are not deleted.
+ #
+ # 100000 is a reasonable batch size for most sites.
+ MaxDeleteBatch: 0
+
+ # Attributes to suppress in events and audit logs. Notably,
+ # specifying {"manifest_text": {}} here typically makes the database
+ # smaller and faster.
+ #
+ # Warning: Using any non-empty value here can have undesirable side
+ # effects for any client or component that relies on event logs.
+ # Use at your own risk.
+ UnloggedAttributes: {}
</pre>
-The above setting affects all events being logged, independently of how much time they will be kept on the database.
-
-<pre>
-# Time to keep audit logs (a row in the log table added each time an
-# Arvados object is created, modified, or deleted) in the PostgreSQL
-# database. Currently, websocket event notifications rely on audit
-# logs, so this should not be set lower than 300 (5 minutes).
-max_audit_log_age: 1209600
-</pre>
-
-...and to prevent surprises and avoid bad database behavior (especially the first time the cleanup job runs on an existing cluster with a huge backlog) a maximum number of rows to delete in a single transaction.
-
-<pre>
-# Maximum number of log rows to delete in a single SQL transaction.
-#
-# If max_audit_log_delete_batch is 0, log entries will never be
-# deleted by Arvados. Cleanup can be done by an external process
-# without affecting any Arvados system processes, as long as very
-# recent (<5 minutes old) logs are not deleted.
-#
-# 100000 is a reasonable batch size for most sites.
-max_audit_log_delete_batch: 0
-</pre>
-
-This feature works when both settings are non-zero, periodically dispatching a background task that deletes all log rows older than @max_audit_log_age@.
-The events being cleaned up by this process don't include job/container stderr logs (they're handled by the existing @delete job/container logs@ rake tasks)
h3. Additional consideration
The storage classes for each volume are set in the per-volume "keepstore configuration":{{site.baseurl}}/install/install-keepstore.html
<pre>
-Volumes:
- - ... Volume configuration ...
- #
- # If no storage classes are specified, will use [default]
- #
- StorageClasses: null
-
- - ... Volume configuration ...
- #
- # Specify this volume is in the "archival" storage class.
- #
- StorageClasses: [archival]
-
+ Volumes:
+ ClusterID-nyw5e-000000000000000:
+ # This volume is in the "default" storage class.
+ StorageClasses:
+ default: true
+ ClusterID-nyw5e-000000000000001:
+ # Specify this volume is in the "archival" storage class.
+ StorageClasses:
+ archival: true
</pre>
Names of storage classes are internal to the cluster and decided by the administrator. Aside from "default", Arvados currently does not define any standard storage class names.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-The "containers" API is the recommended way to submit compute work to Arvados. It supersedes the "jobs" API, which is end-of-life as of Arvados 1.5.
+The "containers" API is the recommended way to submit compute work to Arvados. It supersedes the "jobs" API, which is end-of-life in Arvados 2.0.
h2. Benefits over the "jobs" API
h3(#master). development master (as of 2020-01-16)
+h4. New property vocabulary format for Workbench2
+
+(feature "#14151":https://dev.arvados.org/issues/14151) Workbench2 supports a new vocabulary format and it isn't compatible with the previous one, please read the "workbench2 vocabulary format admin page":{{site.baseurl}}/admin/workbench2-vocabulary.html for more information.
+
h4. "/" prohibited in collection and project names
(issue "#15836":https://dev.arvados.org/issues/15836) By default, Arvados now rejects new names containing the @/@ character when creating or renaming collections and projects. Previously, these names were permitted, but the resulting objects were invisible in the WebDAV "home" tree. If you prefer, you can restore the previous behavior, and optionally configure a substitution string to make the affected objects accessible via WebDAV. See @ForwardSlashNameSubstitution@ in the "configuration reference":config.html.
---
layout: default
navsection: admin
-title: Workbench2 Vocabulary Format
+title: User properties vocabulary
...
{% comment %}
{% codeblock as python %}
{% include 'vocabulary_migrate_py' %}
-{% endcodeblock %}
\ No newline at end of file
+{% endcodeblock %}
* Have filters only matching @[["uuid", "in", [...]]@ or @["uuid", "=", "..."]@
* Specify @count=none@
-* If @select@ is specified, it must include @uuid@
* Not specify @limit@, @offset@ or @order@
* Not request more items than the maximum response size
See "permission links":{{site.baseurl}}/api/permission-model.html#links section of the permission model.
+h3. star
+
+A **star** link is a shortcut to a project that is displayed in the user interface (Workbench) as "favorites". Users can mark their own favorites (implemented by creating or deleting **star** links).
+
+An admin can also create **star** links owned by the "All Users" group, these will be displayed to all users that have permission to read the project that has been favorited.
+
+The schema for a star link is:
+
+table(table table-bordered table-condensed).
+|_. Field|_. Value|_. Description|
+|owner_uuid|user or group uuid|Either the user that owns the favorite, or the "All Users" group for public favorites.|
+|head_uuid|project uuid|The project being favorited|
+|link_class|string of value "star"|Indicates this represents a link to a user favorite|
+
+h4. Creating a favorite
+
+@owner_uuid@ is either an individual user, or the "All Users" group. The @head_uuid@ is the project being favorited.
+
+<pre>
+$ arv link create --link '{
+ "owner_uuid": "zzzzz-j7d0g-fffffffffffffff",
+ "head_uuid": "zzzzz-j7d0g-theprojectuuid",
+ "link_class": "star"}'
+</pre>
+
+h4. Deleting a favorite
+
+<pre>
+$ arv link delete --uuid zzzzz-o0j2j-thestarlinkuuid
+</pre>
+
+h4. Listing favorites
+
+To list all 'star' links that will be displayed for a user:
+
+<pre>
+$ arv link list --filters '[
+ ["link_class", "=", "star"],
+ ["owner_uuid", "in", ["zzzzz-j7d0g-fffffffffffffff", "zzzzz-tpzed-currentuseruuid"]]]'
+</pre>
+
h3. tag
-A **tag** link describes an object using an unparsed plain text string. Tags can be used to annotate objects that are not editable, like collections and objects shared as read-only.
+A **tag** link describes an object using an unparsed plain text string. Tags can be used to annotate objects that are not directly editable by the user, like collections and objects shared as read-only.
table(table table-bordered table-condensed).
|_. tail_type→head_type|_. name→head_uuid {properties}|
---
layout: default
navsection: installguide
-title: Install Workbench2 (beta)
+title: Install Workbench 2
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
Workbench2 is the web-based user interface for Arvados.
{% include 'notebox_begin' %}
-Workbench2 is the replacement for Arvados Workbench. Workbench2 is currently in <i>beta</i>, it is not yet feature complete.
+Workbench2 is the replacement for Arvados Workbench. Workbench2 is suitable for day-to-day use, but does not yet implement every feature of the traditional Workbench.
{% include 'notebox_end' %}
h2(#configure). Update config.yml
//
// * len(Order)==0
//
-// * Each filter must be either "uuid = ..." or "uuid in [...]".
+// * Each filter is either "uuid = ..." or "uuid in [...]".
//
// * The maximum possible response size (total number of objects that
// could potentially be matched by all of the specified filters)
}
}
- if len(todoByRemote) > 1 {
- if cannotSplit {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query: each filter must be either 'uuid = ...' or 'uuid in [...]'")
- }
- if opts.Count != "none" {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query unless count==\"none\"")
- }
- if opts.Limit >= 0 || opts.Offset != 0 || len(opts.Order) > 0 {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with limit, offset, or order parameter")
- }
- if max := conn.cluster.API.MaxItemsPerResponse; nUUIDs > max {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query because number of UUIDs (%d) exceeds page size limit %d", nUUIDs, max)
- }
- selectingUUID := false
- for _, attr := range opts.Select {
- if attr == "uuid" {
- selectingUUID = true
- }
- }
- if opts.Select != nil && !selectingUUID {
- return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with a select parameter that does not include uuid")
- }
+ if len(todoByRemote) == 0 {
+ return nil
+ }
+ if len(todoByRemote) == 1 && todoByRemote[conn.cluster.ClusterID] != nil {
+ // All UUIDs are local, so proxy a single request. The
+ // generic case has some limitations (see below) which
+ // we don't want to impose on local requests.
+ _, err := fn(ctx, conn.cluster.ClusterID, conn.local, opts)
+ return err
+ }
+ if cannotSplit {
+ return httpErrorf(http.StatusBadRequest, "cannot execute federated list query: each filter must be either 'uuid = ...' or 'uuid in [...]'")
+ }
+ if opts.Count != "none" {
+ return httpErrorf(http.StatusBadRequest, "cannot execute federated list query unless count==\"none\"")
+ }
+ if opts.Limit >= 0 || opts.Offset != 0 || len(opts.Order) > 0 {
+ return httpErrorf(http.StatusBadRequest, "cannot execute federated list query with limit, offset, or order parameter")
+ }
+ if max := conn.cluster.API.MaxItemsPerResponse; nUUIDs > max {
+ return httpErrorf(http.StatusBadRequest, "cannot execute federated list query because number of UUIDs (%d) exceeds page size limit %d", nUUIDs, max)
}
ctx, cancel := context.WithCancel(ctx)
return
}
remoteOpts := opts
+ if remoteOpts.Select != nil {
+ // We always need to select UUIDs to
+ // use the response, even if our
+ // caller doesn't.
+ remoteOpts.Select = append([]string{"uuid"}, remoteOpts.Select...)
+ }
for len(todo) > 0 {
if len(batch) > len(todo) {
// Reduce batch to just the todo's
"context"
"fmt"
"net/http"
+ "reflect"
"sort"
"git.arvados.org/arvados.git/sdk/go/arvados"
break
}
if cl.matchFilters(c, options.Filters) {
+ if reflect.DeepEqual(options.Select, []string{"uuid", "name"}) {
+ c = arvados.Collection{UUID: c.UUID, Name: c.Name}
+ } else if reflect.DeepEqual(options.Select, []string{"name"}) {
+ c = arvados.Collection{Name: c.Name}
+ } else if len(options.Select) > 0 {
+ panic(fmt.Sprintf("not implemented: options=%#v", options))
+ }
resp.Items = append(resp.Items, c)
}
}
offset int
order []string
filters []arvados.Filter
+ selectfields []string
expectUUIDs []string
expectCalls []int // number of API calls to backends
expectStatus int
})
}
+func (s *CollectionListSuite) TestCollectionListOneLocalDeselectingUUID(c *check.C) {
+ s.test(c, listTrial{
+ count: "none",
+ limit: -1,
+ filters: []arvados.Filter{{"uuid", "=", s.uuids[0][0]}},
+ selectfields: []string{"name"},
+ expectUUIDs: []string{""}, // select=name is honored
+ expectCalls: []int{1, 0, 0},
+ })
+}
+
func (s *CollectionListSuite) TestCollectionListOneLocalUsingInOperator(c *check.C) {
s.test(c, listTrial{
count: "none",
})
}
+func (s *CollectionListSuite) TestCollectionListOneRemoteDeselectingUUID(c *check.C) {
+ s.test(c, listTrial{
+ count: "none",
+ limit: -1,
+ filters: []arvados.Filter{{"uuid", "=", s.uuids[1][0]}},
+ selectfields: []string{"name"},
+ expectUUIDs: []string{s.uuids[1][0]}, // uuid is returned, despite not being selected
+ expectCalls: []int{0, 1, 0},
+ })
+}
+
func (s *CollectionListSuite) TestCollectionListOneLocalOneRemote(c *check.C) {
s.test(c, listTrial{
count: "none",
})
}
+func (s *CollectionListSuite) TestCollectionListOneLocalOneRemoteDeselectingUUID(c *check.C) {
+ s.test(c, listTrial{
+ count: "none",
+ limit: -1,
+ filters: []arvados.Filter{{"uuid", "in", []string{s.uuids[0][0], s.uuids[1][0]}}},
+ selectfields: []string{"name"},
+ expectUUIDs: []string{s.uuids[0][0], s.uuids[1][0]}, // uuid is returned, despite not being selected
+ expectCalls: []int{1, 1, 0},
+ })
+}
+
func (s *CollectionListSuite) TestCollectionListTwoRemotes(c *check.C) {
s.test(c, listTrial{
count: "none",
Offset: trial.offset,
Order: trial.order,
Filters: trial.filters,
+ Select: trial.selectfields,
})
if trial.expectStatus != 0 {
c.Assert(err, check.NotNil)
[Service]
Type=notify
ExecStart=/usr/bin/crunch-dispatch-slurm
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
LimitNOFILE=1000000
[Service]
Type=simple
ExecStart=/usr/bin/arvados-health
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
[Service]
Type=simple
ExecStart=/usr/bin/keep-balance -commit-pulls -commit-trash
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=10s
Nice=19
[Service]
Type=notify
ExecStart=/usr/bin/keep-web
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
[Service]
Type=notify
ExecStart=/usr/bin/keepproxy
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
Environment=GOGC=10
Type=notify
ExecStart=/usr/bin/keepstore
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
[Service]
Type=notify
ExecStart=/usr/bin/arvados-ws
+# Set a reasonable default for the open file limit
+LimitNOFILE=65536
Restart=always
RestartSec=1
projects / arvados.git / commitdiff