Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>
tag.value.map(function() { vnode.state.dirty(true) })
tag.name.map(m.redraw)
},
+ fixTag: function(vnode, tagName) {
+ // Recover tag if deleted, recover its value if modified
+ savedTagValue = vnode.state.saved_tags[tagName]
+ if (savedTagValue === undefined) {
+ return
+ }
+ found = false
+ vnode.state.tags.forEach(function(tag) {
+ if (tag.name == tagName) {
+ tag.value = vnode.state.saved_tags[tagName]
+ found = true
+ }
+ })
+ if (!found) {
+ vnode.state.tags.pop() // Remove the last empty row
+ vnode.state.appendTag(vnode, tagName, savedTagValue)
+ }
+ },
oninit: function(vnode) {
vnode.state.sessionDB = new SessionDB()
// Get vocabulary
m.request('/vocabulary.json?v=' + vocabularyTimestamp).then(vnode.state.vocabulary)
vnode.state.editMode = vnode.attrs.targetEditable
vnode.state.tags = []
+ vnode.state.saved_tags = {}
vnode.state.dirty = m.stream(false)
vnode.state.dirty.map(m.redraw)
+ vnode.state.error = m.stream('')
vnode.state.objPath = 'arvados/v1/' + vnode.attrs.targetController + '/' + vnode.attrs.targetUuid
// Get tags
vnode.state.sessionDB.request(
}
// Data synced with server, so dirty state should be false
vnode.state.dirty(false)
+ vnode.state.saved_tags = o.properties
// Add new tag row when the last one is completed
vnode.state.dirty.map(function() {
if (!vnode.state.editMode) { return }
}
).then(function(v) {
vnode.state.dirty(false)
+ vnode.state.error('')
+ vnode.state.saved_tags = tags
+ }).catch(function(err) {
+ if (err.errors !== undefined) {
+ var re = /protected\ property/i
+ var protected_props = []
+ err.errors.forEach(function(error) {
+ if (re.test(error)) {
+ prop = error.split(':')[1].trim()
+ vnode.state.fixTag(vnode, prop)
+ protected_props.push(prop)
+ }
+ })
+ if (protected_props.length > 0) {
+ errMsg = "Protected properties cannot be updated: " + protected_props.join(', ')
+ } else {
+ errMsg = errors.join(', ')
+ }
+ } else {
+ errMsg = err
+ }
+ vnode.state.error(errMsg)
})
}
- }, vnode.state.dirty() ? ' Save changes ' : ' Saved ')
+ }, vnode.state.dirty() ? ' Save changes ' : ' Saved '),
+ m('span', {
+ style: {
+ color: '#ff0000',
+ margin: '0px 10px'
+ }
+ }, [ vnode.state.error() ])
]),
// Tags table
m(TagEditorTable, {
- Cloud:
- admin/storage-classes.html.textile.liquid
- admin/spot-instances.html.textile.liquid
- - Other:
+ - Data Management:
- admin/collection-versioning.html.textile.liquid
+ - admin/collection-managed-properties.html.textile.liquid
+ - Other:
- admin/federation.html.textile.liquid
- admin/controlling-container-reuse.html.textile.liquid
- admin/logs-table-management.html.textile.liquid
--- /dev/null
+#!/usr/bin/env python
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+import arvados
+import arvados.util as util
+
+filters = [['properties.responsible_person_uuid', 'exists', False]]
+cols = util.list_all(arvados.api().collections().list, filters=filters, select=['uuid', 'name'])
+
+print("Found {} collections:".format(len(cols)))
+for c in cols:
+ print('{}, "{}"'.format(c['uuid'], c['name']))
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env python
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+import arvados
+import arvados.util as util
+
+def get_subproject_uuids(api, root_uuid):
+ uuids = []
+ groups = util.list_all(api.groups().list, filters=[['owner_uuid', '=', '{}'.format(root_uuid)]], select=['uuid'])
+ for g in groups:
+ uuids += ([g['uuid']] + get_subproject_uuids(api, g['uuid']))
+ return uuids
+
+def get_cols(api, filters):
+ cols = util.list_all(api.collections().list, filters=filters, select=['uuid', 'properties'])
+ return cols
+
+# Search for collections on project hierarchy rooted at root_uuid
+root_uuid = 'zzzzz-j7d0g-ppppppppppppppp'
+# Set the property to the UUID below
+responsible_uuid = 'zzzzz-tpzed-xxxxxxxxxxxxxxx'
+
+api = arvados.api()
+for p_uuid in [root_uuid] + get_subproject_uuids(api, root_uuid):
+ f = [['properties.responsible_person_uuid', 'exists', False],
+ ['owner_uuid', '=', p_uuid]]
+ cols = get_cols(api, f)
+ print("Found {} collections owned by {}".format(len(cols), p_uuid))
+ for c in cols:
+ print(" - Updating collection {}".format(c["uuid"]))
+ props = c['properties']
+ props['responsible_person_uuid'] = responsible_uuid
+ api.collections().update(uuid=c['uuid'], body={'properties': props}).execute()
\ No newline at end of file
--- /dev/null
+#!/usr/bin/env python
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+import arvados
+import arvados.util as util
+
+old_uuid = "zzzzz-tpzed-xxxxxxxxxxxxxxx"
+new_uuid = "zzzzz-tpzed-yyyyyyyyyyyyyyy"
+
+api = arvados.api()
+filters = [['properties.responsible_person_uuid', '=', '{}'.format(old_uuid)]]
+cols = util.list_all(api.collections().list, filters=filters, select=['uuid', 'properties'])
+
+print("Found {} collections".format(len(cols)))
+for c in cols:
+ print("Updating collection {}".format(c["uuid"]))
+ props = c['properties']
+ props['responsible_person_uuid'] = new_uuid
+ api.collections().update(uuid=c['uuid'], body={'properties': props}).execute()
\ No newline at end of file
h2. Download and tag the latest arvados/jobs docker image
-In order to start workflows from workbench, there needs to be Docker image tagged @arvados/jobs:latest@. The following command downloads the latest arvados/jobs image from Docker Hub, loads it into Keep, and tags it as 'latest'. In this example @$project_uuid@ should be the the UUID of the "Arvados Standard Docker Images" project.
+In order to start workflows from workbench, there needs to be Docker image tagged @arvados/jobs:latest@. The following command downloads the latest arvados/jobs image from Docker Hub, loads it into Keep, and tags it as 'latest'. In this example @$project_uuid@ should be the UUID of the "Arvados Standard Docker Images" project.
<notextile>
<pre><code>~$ <span class="userinput">arv-keepdocker --pull arvados/jobs latest --project-uuid $project_uuid</span>
h2(#symlinks-in-output). Symlinks in output
-When a container's output_path is a tmp mount backed by local disk, this output directory can contain symlinks to other files in the the output directory, or to collection mount points. If the symlink leads to a collection mount, efficiently copy the collection into the output collection. Symlinks leading to files or directories are expanded and created as regular files in the output collection. Further, whether symlinks are relative or absolute, every symlink target (even targets that are symlinks themselves) must point to a path in either the output directory or a collection mount.
+When a container's output_path is a tmp mount backed by local disk, this output directory can contain symlinks to other files in the output directory, or to collection mount points. If the symlink leads to a collection mount, efficiently copy the collection into the output collection. Symlinks leading to files or directories are expanded and created as regular files in the output collection. Further, whether symlinks are relative or absolute, every symlink target (even targets that are symlinks themselves) must point to a path in either the output directory or a collection mount.
--- /dev/null
+---
+layout: default
+navsection: admin
+title: Configuring collection's managed properties
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Collection's managed properties allow a cluster administrator to enable some special behaviors regarding properties at creation & update times.
+This page describes how to enable and configure these behaviors on the API server.
+
+h3. API Server configuration
+
+The @Collections.ManagedProperties@ setting from the @config.yml@ file is used for enabling any of the following behaviors:
+
+h4. Pre-assigned property key & value
+
+For every newly created collection, assign a predefined key/value pair if it isn't already passed at creation time:
+
+<pre>
+Collections:
+ ManagedProperties:
+ foo: {value: bar}
+</pre>
+
+h4. Original owner UUID
+
+This behavior will assign to a property key the UUID of the user who owns the collection's contaning project.
+
+<pre>
+Collections:
+ ManagedProperties:
+ responsible_person_uuid: {function: original_owner}
+</pre>
+
+h4. Protected properties
+
+If there's a need to prevent a non-admin user from modifying a specific property, even by its owner, the @protected@ attribute can be set to @true@, like so:
+
+<pre>
+Collections:
+ ManagedProperties:
+ responsible_person_uuid: {function: original_owner, protected: true}
+</pre>
+
+This property can be applied to any of the defined managed properties. If missing, it's assumed as being @false@ by default.
+
+h3. Supporting example scripts
+
+When enabling this feature, there may be pre-existing collections that won't have the managed properties just configured. The following script examples may be helpful to sync these older collections.
+
+For the following examples we assume that the @responsible_person_uuid@ property is set as @{function: original_owner, protected: true}@.
+
+h4. List uuid/names of collections without @responsible_person_uuid@ property
+
+The collection's managed properties feature assigns the configured properties to newly created collections. This means that previously existing collections won't get the default properties and if needed, they should be assigned manually.
+
+The following example script outputs a listing of collection UUIDs and names of those collections that don't include the @responsible_person_uuid@ property.
+
+{% codeblock as python %}
+{% include 'admin_list_collections_without_property_py' %}
+{% endcodeblock %}
+
+h4. Update the @responsible_person_uuid@ property from nil to X in the project hierarchy rooted at P
+
+When enabling @responsible_person_uuid@, new collections will get this property's value set to the user who owns the root project where the collection is placed, but older collections won't have the property set. The following example script allows an administrator to set the @responsible_person_uuid@ property to collections below a certaing project hierarchy.
+
+{% codeblock as python %}
+{% include 'admin_set_property_to_collections_under_project_py' %}
+{% endcodeblock %}
+
+h4. Update the @responsible_person_uuid@ property from X to Y on all collections
+
+This example can be useful to change responsibility from one user to another.
+
+Please note that the following code should run with admin privileges, assuming that the managed property is @protected@.
+
+{% codeblock as python %}
+{% include 'admin_update_collection_property_py' %}
+{% endcodeblock %}
+
Two Arvados Rails servers store data in a PostgreSQL database: the SSO server, and the API server. The API server requires at least version *9.4* of PostgreSQL. Beyond that, you have the flexibility to deploy PostgreSQL any way that the Rails servers will be able to connect to it. Our recommended deployment strategy is:
-* Install PostgreSQL on the the same host as the SSO server, and dedicate that install to hosting the SSO database. This provides the best security for the SSO server, because the database does not have to accept any client connections over the network. Typical load on the SSO server is light enough that deploying both it and its database on the same host does not compromise performance.
+* Install PostgreSQL on the same host as the SSO server, and dedicate that install to hosting the SSO database. This provides the best security for the SSO server, because the database does not have to accept any client connections over the network. Typical load on the SSO server is light enough that deploying both it and its database on the same host does not compromise performance.
* If you want to provide the most scalability for your Arvados cluster, install PostgreSQL for the API server on a dedicated host. This gives you the most flexibility to avoid resource contention, and tune performance separately for the API server and its database. If performance is less of a concern for your installation, you can install PostgreSQL on the API server host directly, as with the SSO server.
Find the section for your distribution below, and follow it to install PostgreSQL on each host where you will deploy it. Then follow the steps in the later section(s) to set up PostgreSQL for the Arvados service(s) that need it.
* "Java SDK v2":{{site.baseurl}}/sdk/java-v2/index.html
* "Java SDK v1":{{site.baseurl}}/sdk/java/index.html
-Many Arvados Workbench pages, under the the *Advanced* tab, provide examples of API and SDK use for accessing the current resource .
+Many Arvados Workbench pages, under the *Advanced* tab, provide examples of API and SDK use for accessing the current resource .
When collection versioning is enabled, updating certain collection attributes (@name@, @description@, @properties@, @manifest_text@) will save a copy of the collection state, previous to the update. This copy (a new collection record) will have its own @uuid@, and a @current_version_uuid@ attribute pointing to the current version's @uuid@.
-Every collection has a @version@ attribute that indicates its version number, starting from 1 on new collections and incrementing by 1 with every versionable update. All collections point to their most current version via the @current_version_uuid@ attribute, being @uuid@ and @current_version_uuid@ equal on those collection records that are the the current version of themselves. Note that the "current version" collection record doesn't change its @uuid@, "past versions" are saved as new records every time it's needed, pointing to the current collection record.
+Every collection has a @version@ attribute that indicates its version number, starting from 1 on new collections and incrementing by 1 with every versionable update. All collections point to their most current version via the @current_version_uuid@ attribute, being @uuid@ and @current_version_uuid@ equal on those collection records that are the current version of themselves. Note that the "current version" collection record doesn't change its @uuid@, "past versions" are saved as new records every time it's needed, pointing to the current collection record.
A version will be saved when one of the following conditions is true:
# > 0s = auto-create a new version when older than the specified number of seconds.
PreserveVersionIfIdle: -1s
+ # Managed collection properties. At creation time, if the client didn't
+ # provide the listed keys, they will be automatically populated following
+ # one of the following behaviors:
+ #
+ # * UUID of the user who owns the containing project.
+ # responsible_person_uuid: {function: original_owner, protected: true}
+ #
+ # * Default concrete value.
+ # foo_bar: {value: baz, protected: false}
+ #
+ # If protected is true, only an admin user can modify its value.
+ ManagedProperties: {}
+
Login:
# These settings are provided by your OAuth2 provider (e.g.,
# sso-provider).
# > 0s = auto-create a new version when older than the specified number of seconds.
PreserveVersionIfIdle: -1s
+ # Managed collection properties. At creation time, if the client didn't
+ # provide the listed keys, they will be automatically populated following
+ # one of the following behaviors:
+ #
+ # * UUID of the user who owns the containing project.
+ # responsible_person_uuid: {function: original_owner, protected: true}
+ #
+ # * Default concrete value.
+ # foo_bar: {value: baz, protected: false}
+ #
+ # If protected is true, only an admin user can modify its value.
+ ManagedProperties: {}
+
Login:
# These settings are provided by your OAuth2 provider (e.g.,
# sso-provider).
}
updatedReq.Header.Set("Authorization", "Bearer "+token)
- // Remove api_token=... from the the query string, in case we
+ // Remove api_token=... from the query string, in case we
// end up forwarding the request.
if values, err := url.ParseQuery(updatedReq.URL.RawQuery); err != nil {
return nil, err
public static void main(String[] args){
System.out.println("Welcome to Arvados Java SDK.");
- System.out.println("Please refer to http://doc.arvados.org/sdk/java/index.html to get started with the the SDK.");
+ System.out.println("Please refer to http://doc.arvados.org/sdk/java/index.html to get started with the SDK.");
}
}
end
end
+ def user_owner_uuid
+ if self.owner_uuid.nil?
+ return current_user.uuid
+ end
+ owner_class = ArvadosModel.resource_class_for_uuid(self.owner_uuid)
+ if owner_class == User
+ self.owner_uuid
+ else
+ owner_class.find_by_uuid(self.owner_uuid).user_owner_uuid
+ end
+ end
+
def logged_attributes
attributes.except(*Rails.configuration.AuditLogs.UnloggedAttributes)
end
before_validation :default_empty_manifest
before_validation :default_storage_classes, on: :create
+ before_validation :default_properties, on: :create
before_validation :check_encoding
before_validation :check_manifest_validity
before_validation :check_signatures
validate :ensure_storage_classes_contain_non_empty_strings
validate :versioning_metadata_updates, on: :update
validate :past_versions_cannot_be_updated, on: :update
+ validate :protected_default_properties_updates, on: :update
after_validation :set_file_count_and_total_size
before_save :set_file_names
around_update :manage_versioning, unless: :is_past_version?
self.storage_classes_confirmed ||= []
end
+ # Sets default properties at creation time
+ def default_properties
+ default_props = Rails.configuration.Collections.ManagedProperties.with_indifferent_access
+ if default_props.empty?
+ return
+ end
+ (default_props.keys - self.properties.keys).each do |key|
+ if default_props[key].has_key?('value')
+ self.properties[key] = default_props[key]['value']
+ elsif default_props[key]['function'].andand == 'original_owner'
+ self.properties[key] = self.user_owner_uuid
+ else
+ logger.warn "Unidentified default property definition '#{key}': #{default_props[key].inspect}"
+ end
+ end
+ end
+
def portable_manifest_text
self.class.munge_manifest_locators(manifest_text) do |match|
if match[2] # size
end
end
+ def protected_default_properties_updates
+ default_properties = Rails.configuration.Collections.ManagedProperties.with_indifferent_access
+ if default_properties.empty? || !properties_changed? || current_user.is_admin
+ return true
+ end
+ protected_props = default_properties.keys.select do |p|
+ Rails.configuration.Collections.ManagedProperties[p]['protected']
+ end
+ # Pre-existent protected properties can't be updated
+ invalid_updates = properties_was.keys.select{|p| properties_was[p] != properties[p]} & protected_props
+ if !invalid_updates.empty?
+ invalid_updates.each do |p|
+ errors.add("protected property cannot be updated:", p)
+ end
+ raise PermissionDeniedError.new
+ end
+ true
+ end
+
def versioning_metadata_updates
valid = true
if !is_past_version? && current_version_uuid_changed?
SweepTrashedObjects.sweep_now
assert_empty Collection.where(uuid: uuid)
end
+
+ test "create collections with default properties" do
+ Rails.configuration.Collections.ManagedProperties = {
+ 'default_prop1' => {'value' => 'prop1_value'},
+ 'responsible_person_uuid' => {'function' => 'original_owner'}
+ }
+ # Test collection without initial properties
+ act_as_user users(:active) do
+ c = create_collection 'foo', Encoding::US_ASCII
+ assert c.valid?
+ assert_not_empty c.properties
+ assert_equal 'prop1_value', c.properties['default_prop1']
+ assert_equal users(:active).uuid, c.properties['responsible_person_uuid']
+ end
+ # Test collection with default_prop1 property already set
+ act_as_user users(:active) do
+ c = Collection.create(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n",
+ properties: {'default_prop1' => 'custom_value'})
+ assert c.valid?
+ assert_not_empty c.properties
+ assert_equal 'custom_value', c.properties['default_prop1']
+ assert_equal users(:active).uuid, c.properties['responsible_person_uuid']
+ end
+ # Test collection inside a sub project
+ act_as_user users(:active) do
+ c = Collection.create(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n",
+ owner_uuid: groups(:asubproject).uuid)
+ assert c.valid?
+ assert_not_empty c.properties
+ assert_equal users(:active).uuid, c.properties['responsible_person_uuid']
+ end
+ end
+
+ test "update collection with protected default properties" do
+ Rails.configuration.Collections.ManagedProperties = {
+ 'default_prop1' => {'value' => 'prop1_value', 'protected' => true},
+ }
+ act_as_user users(:active) do
+ c = create_collection 'foo', Encoding::US_ASCII
+ assert c.valid?
+ assert_not_empty c.properties
+ assert_equal 'prop1_value', c.properties['default_prop1']
+ # Add new property
+ c.properties['prop2'] = 'value2'
+ c.save!
+ c.reload
+ assert_equal 'value2', c.properties['prop2']
+ # Try to change protected property's value
+ c.properties['default_prop1'] = 'new_value'
+ assert_raises(ArvadosModel::PermissionDeniedError) do
+ c.save!
+ end
+ # Admins are allowed to change protected properties
+ act_as_system_user do
+ c.properties['default_prop1'] = 'new_value'
+ c.save!
+ c.reload
+ assert_equal 'new_value', c.properties['default_prop1']
+ end
+ end
+ end
end
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
-
check "gopkg.in/check.v1"
)
// srvList returns the KeepServices, sorted in rendezvous order and
// then selected by idx. For example, srvList(3, slots{0, 1, 4})
-// returns the the first-, second-, and fifth-best servers for storing
+// returns the first-, second-, and fifth-best servers for storing
// bal.knownBlkid(3).
func (bal *balancerSuite) srvList(knownBlockID int, order slots) (srvs []*KeepService) {
for _, i := range order {
real_sigaction(SIGINT, act, oldact);
}
- // Forward the call the the real sigaction.
+ // Forward the call to the real sigaction.
return real_sigaction(signum, act, oldact);
}
projects / arvados.git / commitdiff