lib/cli
lib/cmd
lib/controller
+lib/controller/federation
+lib/controller/railsproxy
+lib/controller/router
+lib/controller/rpc
lib/crunchstat
lib/cloud
lib/cloud/azure
+ lib/cloud/cloudtest
lib/dispatchcloud
lib/dispatchcloud/container
lib/dispatchcloud/scheduler
ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
go get -v github.com/kardianos/govendor
cd "$GOPATH/src/git.curoverse.com/arvados.git"
- if [[ -n "$short" ]]; then
- go get -v -d ...
- "$GOPATH/bin/govendor" sync
- else
- # Remove cached source dirs in workdir. Otherwise, they will
- # not qualify as +missing or +external below, and we won't be
- # able to detect that they're missing from vendor/vendor.json.
- rm -rf vendor/*/
- go get -v -d ...
- "$GOPATH/bin/govendor" sync
- [[ -z $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]] \
- || fatal "vendor/vendor.json has unused or missing dependencies -- try:
-
- (export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused)
-
- ";
- fi
+ go get -v -d ...
+ "$GOPATH/bin/govendor" sync
) || fatal "Go setup failed"
setup_virtualenv "$VENVDIR" --python python2.7
services/api)
stop_services
;;
- gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+ gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
# don't care whether services are running
;;
*)
)
declare -a gostuff
-gostuff=(
- cmd/arvados-client
- cmd/arvados-server
- lib/cli
- lib/cmd
- lib/controller
- lib/crunchstat
- lib/cloud
- lib/cloud/azure
- lib/cloud/ec2
- lib/cloud/cloudtest
- lib/config
- lib/dispatchcloud
- lib/dispatchcloud/container
- lib/dispatchcloud/scheduler
- lib/dispatchcloud/ssh_executor
- lib/dispatchcloud/worker
- lib/service
- sdk/go/arvados
- sdk/go/arvadosclient
- sdk/go/auth
- sdk/go/blockdigest
- sdk/go/dispatch
- sdk/go/health
- sdk/go/httpserver
- sdk/go/manifest
- sdk/go/asyncbuf
- sdk/go/crunchrunner
- sdk/go/stats
- services/arv-git-httpd
- services/crunchstat
- services/health
- services/keep-web
- services/keepstore
- sdk/go/keepclient
- services/keep-balance
- services/keepproxy
- services/crunch-dispatch-local
- services/crunch-dispatch-slurm
- services/crunch-run
- services/ws
- tools/keep-block-check
- tools/keep-exercise
- tools/keep-rsync
- tools/sync-groups
-)
+gostuff=($(git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u))
install_apps/workbench() {
cd "$WORKSPACE/apps/workbench" \
[[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
}
+ test_govendor() {
+ (
+ set -e
+ cd "$GOPATH/src/git.curoverse.com/arvados.git"
+ # Remove cached source dirs in workdir. Otherwise, they will
+ # not qualify as +missing or +external below, and we won't be
+ # able to detect that they're missing from vendor/vendor.json.
+ rm -rf vendor/*/
+ go get -v -d ...
+ "$GOPATH/bin/govendor" sync
+ if [[ -n $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]]; then
+ echo >&2 "vendor/vendor.json has unused or missing dependencies -- try:
+
+ (export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused)
+
+ "
+ return 1
+ fi
+ )
+ }
+
test_services/api() {
rm -f "$WORKSPACE/services/api/git-commit.version"
cd "$WORKSPACE/services/api" \
fi
do_test gofmt
+ do_test govendor
do_test doc
do_test sdk/ruby
do_test sdk/R
# assume emacs, or something, is offering a history buffer
# and pre-populating the command will only cause trouble
nextcmd=
- elif [[ "$nextcmd" != "install deps" ]]; then
- :
- elif [[ -e "$VENVDIR/bin/activate" ]]; then
- nextcmd="test lib/cmd"
- else
+ elif [[ ! -e "$VENVDIR/bin/activate" ]]; then
nextcmd="install deps"
+ else
+ nextcmd=""
fi
}
echo
help_interactive
nextcmd="install deps"
setnextcmd
- while read -p 'What next? ' -e -i "${nextcmd}" nextcmd; do
+ HISTFILE="$WORKSPACE/tmp/.history"
+ history -r
+ while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
+ history -s "$nextcmd"
+ history -w
read verb target opts <<<"${nextcmd}"
target="${target%/}"
target="${target/\/:/:}"
# > 0s = auto-create a new version when older than the specified number of seconds.
PreserveVersionIfIdle: -1s
+ # Managed collection properties. At creation time, if the client didn't
+ # provide the listed keys, they will be automatically populated following
+ # one of the following behaviors:
+ #
+ # * UUID of the user who owns the containing project.
+ # responsible_person_uuid: {Function: original_owner, Protected: true}
+ #
+ # * Default concrete value.
+ # foo_bar: {Value: baz, Protected: false}
+ #
+ # If Protected is true, only an admin user can modify its value.
+ ManagedProperties:
+ SAMPLE: {Function: original_owner, Protected: true}
+
Login:
# These settings are provided by your OAuth2 provider (e.g.,
# sso-provider).
# Shell command to execute on each worker to determine whether
# the worker is booted and ready to run containers. It should
# exit zero if the worker is ready.
- BootProbeCommand: "docker ps"
+ BootProbeCommand: "docker ps -q"
# Minimum interval between consecutive probes to a single
# worker.
# Workbench2 configs
VocabularyURL: ""
FileViewersConfigURL: ""
+
+ # Use experimental controller code (see https://dev.arvados.org/issues/14287)
+ EnableBetaController14287: false
"Collections.CollectionVersioning": false,
"Collections.DefaultReplication": true,
"Collections.DefaultTrashLifetime": true,
+ "Collections.ManagedProperties": true,
+ "Collections.ManagedProperties.*": true,
+ "Collections.ManagedProperties.*.*": true,
"Collections.PreserveVersionIfIdle": true,
"Collections.TrashSweepInterval": false,
"Containers": true,
"Containers.StaleLockTimeout": false,
"Containers.SupportedDockerImageFormats": true,
"Containers.UsePreemptibleInstances": true,
+ "EnableBetaController14287": false,
"Git": false,
"InstanceTypes": true,
"InstanceTypes.*": true,
# > 0s = auto-create a new version when older than the specified number of seconds.
PreserveVersionIfIdle: -1s
+ # Managed collection properties. At creation time, if the client didn't
+ # provide the listed keys, they will be automatically populated following
+ # one of the following behaviors:
+ #
+ # * UUID of the user who owns the containing project.
+ # responsible_person_uuid: {Function: original_owner, Protected: true}
+ #
+ # * Default concrete value.
+ # foo_bar: {Value: baz, Protected: false}
+ #
+ # If Protected is true, only an admin user can modify its value.
+ ManagedProperties:
+ SAMPLE: {Function: original_owner, Protected: true}
+
Login:
# These settings are provided by your OAuth2 provider (e.g.,
# sso-provider).
# Shell command to execute on each worker to determine whether
# the worker is booted and ready to run containers. It should
# exit zero if the worker is ready.
- BootProbeCommand: "docker ps"
+ BootProbeCommand: "docker ps -q"
# Minimum interval between consecutive probes to a single
# worker.
# Workbench2 configs
VocabularyURL: ""
FileViewersConfigURL: ""
+
+ # Use experimental controller code (see https://dev.arvados.org/issues/14287)
+ EnableBetaController14287: false
`)
Collections struct {
BlobSigning bool
BlobSigningKey string
- DefaultReplication int
BlobSigningTTL Duration
- DefaultTrashLifetime Duration
- TrashSweepInterval Duration
CollectionVersioning bool
+ DefaultTrashLifetime Duration
+ DefaultReplication int
+ ManagedProperties map[string]interface{}
PreserveVersionIfIdle Duration
+ TrashSweepInterval Duration
}
Git struct {
Repositories string
UserProfileFormMessage string
VocabularyURL string
}
+
+ EnableBetaController14287 bool
}
type Services struct {
before_validation :default_empty_manifest
before_validation :default_storage_classes, on: :create
+ before_validation :managed_properties, on: :create
before_validation :check_encoding
before_validation :check_manifest_validity
before_validation :check_signatures
before_validation :strip_signatures_and_update_replication_confirmed
+ before_validation :name_null_if_empty
validate :ensure_pdh_matches_manifest_text
validate :ensure_storage_classes_desired_is_not_empty
validate :ensure_storage_classes_contain_non_empty_strings
validate :versioning_metadata_updates, on: :update
validate :past_versions_cannot_be_updated, on: :update
+ validate :protected_managed_properties_updates, on: :update
after_validation :set_file_count_and_total_size
before_save :set_file_names
around_update :manage_versioning, unless: :is_past_version?
api_accessible :user, extend: :common do |t|
- t.add :name
+ t.add lambda { |x| x.name || "" }, as: :name
t.add :description
t.add :properties
t.add :portable_data_hash
# correct timestamp in signed_manifest_text.
'manifest_text' => ['manifest_text', 'trash_at', 'is_trashed'],
'unsigned_manifest_text' => ['manifest_text'],
+ 'name' => ['name'],
)
end
end
end
+ def name_null_if_empty
+ if name == ""
+ self.name = nil
+ end
+ end
+
def set_file_names
if self.manifest_text_changed?
self.file_names = manifest_files
self.storage_classes_confirmed ||= []
end
+ # Sets managed properties at creation time
+ def managed_properties
+ managed_props = Rails.configuration.Collections.ManagedProperties.with_indifferent_access
+ if managed_props.empty?
+ return
+ end
+ (managed_props.keys - self.properties.keys).each do |key|
+ if managed_props[key].has_key?('Value')
+ self.properties[key] = managed_props[key]['Value']
+ elsif managed_props[key]['Function'].andand == 'original_owner'
+ self.properties[key] = self.user_owner_uuid
+ else
+ logger.warn "Unidentified default property definition '#{key}': #{managed_props[key].inspect}"
+ end
+ end
+ end
+
def portable_manifest_text
self.class.munge_manifest_locators(manifest_text) do |match|
if match[2] # size
end
end
+ def protected_managed_properties_updates
+ managed_properties = Rails.configuration.Collections.ManagedProperties.with_indifferent_access
+ if managed_properties.empty? || !properties_changed? || current_user.is_admin
+ return true
+ end
+ protected_props = managed_properties.keys.select do |p|
+ Rails.configuration.Collections.ManagedProperties[p]['Protected']
+ end
+ # Pre-existent protected properties can't be updated
+ invalid_updates = properties_was.keys.select{|p| properties_was[p] != properties[p]} & protected_props
+ if !invalid_updates.empty?
+ invalid_updates.each do |p|
+ errors.add("protected property cannot be updated:", p)
+ end
+ raise PermissionDeniedError.new
+ end
+ true
+ end
+
def versioning_metadata_updates
valid = true
if !is_past_version? && current_version_uuid_changed?
assert_empty Collection.where(uuid: uuid)
end
+ test "empty names are exempt from name uniqueness" do
+ act_as_user users(:active) do
+ c1 = Collection.new(name: nil, manifest_text: '', owner_uuid: groups(:aproject).uuid)
+ assert c1.save
+ c2 = Collection.new(name: '', manifest_text: '', owner_uuid: groups(:aproject).uuid)
+ assert c2.save
+ c3 = Collection.new(name: '', manifest_text: '', owner_uuid: groups(:aproject).uuid)
+ assert c3.save
+ c4 = Collection.new(name: 'c4', manifest_text: '', owner_uuid: groups(:aproject).uuid)
+ assert c4.save
+ c5 = Collection.new(name: 'c4', manifest_text: '', owner_uuid: groups(:aproject).uuid)
+ assert_raises(ActiveRecord::RecordNotUnique) do
+ c5.save
+ end
+ end
+ end
++
+ test "create collections with managed properties" do
+ Rails.configuration.Collections.ManagedProperties = {
+ 'default_prop1' => {'Value' => 'prop1_value'},
+ 'responsible_person_uuid' => {'Function' => 'original_owner'}
+ }
+ # Test collection without initial properties
+ act_as_user users(:active) do
+ c = create_collection 'foo', Encoding::US_ASCII
+ assert c.valid?
+ assert_not_empty c.properties
+ assert_equal 'prop1_value', c.properties['default_prop1']
+ assert_equal users(:active).uuid, c.properties['responsible_person_uuid']
+ end
+ # Test collection with default_prop1 property already set
+ act_as_user users(:active) do
+ c = Collection.create(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n",
+ properties: {'default_prop1' => 'custom_value'})
+ assert c.valid?
+ assert_not_empty c.properties
+ assert_equal 'custom_value', c.properties['default_prop1']
+ assert_equal users(:active).uuid, c.properties['responsible_person_uuid']
+ end
+ # Test collection inside a sub project
+ act_as_user users(:active) do
+ c = Collection.create(manifest_text: ". d41d8cd98f00b204e9800998ecf8427e 0:34:foo.txt\n",
+ owner_uuid: groups(:asubproject).uuid)
+ assert c.valid?
+ assert_not_empty c.properties
+ assert_equal users(:active).uuid, c.properties['responsible_person_uuid']
+ end
+ end
+
+ test "update collection with protected managed properties" do
+ Rails.configuration.Collections.ManagedProperties = {
+ 'default_prop1' => {'Value' => 'prop1_value', 'Protected' => true},
+ }
+ act_as_user users(:active) do
+ c = create_collection 'foo', Encoding::US_ASCII
+ assert c.valid?
+ assert_not_empty c.properties
+ assert_equal 'prop1_value', c.properties['default_prop1']
+ # Add new property
+ c.properties['prop2'] = 'value2'
+ c.save!
+ c.reload
+ assert_equal 'value2', c.properties['prop2']
+ # Try to change protected property's value
+ c.properties['default_prop1'] = 'new_value'
+ assert_raises(ArvadosModel::PermissionDeniedError) do
+ c.save!
+ end
+ # Admins are allowed to change protected properties
+ act_as_system_user do
+ c.properties['default_prop1'] = 'new_value'
+ c.save!
+ c.reload
+ assert_equal 'new_value', c.properties['default_prop1']
+ end
+ end
+ end
end
projects / arvados.git / commitdiff