Merge branch 'master' of git.clinicalfuture.com:arvados
authorPeter Amstutz <peter.amstutz@curoverse.com>
Mon, 10 Mar 2014 21:04:48 +0000 (17:04 -0400)
committerPeter Amstutz <peter.amstutz@curoverse.com>
Mon, 10 Mar 2014 21:04:48 +0000 (17:04 -0400)
# Please enter a commit message to explain why this merge is necessary,
# especially if it merges an updated upstream into a topic branch.
#
# Lines starting with '#' will be ignored, and an empty message aborts
# the commit.

42 files changed:
apps/workbench/.gitignore
apps/workbench/app/controllers/users_controller.rb
apps/workbench/app/views/users/_show_admin.html.erb [new file with mode: 0644]
apps/workbench/config/application.default.yml [new file with mode: 0644]
apps/workbench/config/application.yml.example [new file with mode: 0644]
apps/workbench/config/initializers/zz_load_config.rb [new file with mode: 0644]
apps/workbench/config/routes.rb
apps/workbench/lib/tasks/config_check.rake [new file with mode: 0644]
doc/api/schema/ApiClientAuthorization.html.textile.liquid
doc/api/schema/AuthorizedKey.html.textile.liquid
doc/api/schema/Commit.html.textile.liquid
doc/api/schema/CommitAncestor.html.textile.liquid
doc/api/schema/Group.html.textile.liquid
doc/api/schema/Human.html.textile.liquid
doc/api/schema/KeepDisk.html.textile.liquid
doc/api/schema/Log.html.textile.liquid
doc/api/schema/Node.html.textile.liquid
doc/api/schema/PipelineInstance.html.textile.liquid
doc/api/schema/PipelineTemplate.html.textile.liquid
doc/api/schema/Repository.html.textile.liquid
doc/api/schema/Specimen.html.textile.liquid
doc/api/schema/Trait.html.textile.liquid
doc/api/schema/User.html.textile.liquid
doc/api/schema/VirtualMachine.html.textile.liquid
doc/install/install-api-server.html.md.liquid
sdk/python/arvados/collection.py
sdk/python/arvados/errors.py
sdk/python/test_collections.py
services/api/.gitignore
services/api/app/controllers/arvados/v1/api_client_authorizations_controller.rb
services/api/app/models/arvados_model.rb
services/api/app/models/blob.rb [new file with mode: 0644]
services/api/app/models/node.rb
services/api/config/application.default.yml [new file with mode: 0644]
services/api/config/application.yml.example [new file with mode: 0644]
services/api/config/database.yml.sample
services/api/config/environments/test.rb.example
services/api/config/initializers/zz_load_config.rb [new file with mode: 0644]
services/api/config/initializers/zz_preload_all_models.rb [moved from services/api/config/initializers/preload_all_models.rb with 100% similarity]
services/api/lib/tasks/config_check.rake [new file with mode: 0644]
services/api/test/integration/api_client_authorizations_api_test.rb
services/api/test/unit/blob_test.rb [new file with mode: 0644]

index 89efb8185f003a5019d99a83a547798fcdca6a0d..a656a5bc61f6dc05b29b840c25084fbfe62a375f 100644 (file)
@@ -22,6 +22,7 @@
 /config/environments/development.rb
 /config/environments/test.rb
 /config/environments/production.rb
+/config/application.yml
 
 /config/piwik.yml
 
index 3ccaa525cee853e43e9cd1f963419638152a53b0..c33de2d034a1630b56a3ca0000aa87440ad82c7a 100644 (file)
@@ -1,6 +1,7 @@
 class UsersController < ApplicationController
   skip_before_filter :find_object_by_uuid, :only => :welcome
   skip_around_filter :thread_with_mandatory_api_token, :only => :welcome
+  before_filter :ensure_current_user_is_admin, only: :sudo
 
   def welcome
     if current_user
@@ -9,6 +10,23 @@ class UsersController < ApplicationController
     end
   end
 
+  def show_pane_list
+    if current_user.andand.is_admin
+      super | %w(Admin)
+    else
+      super
+    end
+  end
+
+  def sudo
+    resp = $arvados_api_client.api(ApiClientAuthorization, '', {
+                                     api_client_authorization: {
+                                       owner_uuid: @object.uuid
+                                     }
+                                   })
+    redirect_to root_url(api_token: resp[:api_token])
+  end
+
   def home
     @showallalerts = false
     @my_ssh_keys = AuthorizedKey.where(authorized_user_uuid: current_user.uuid)
diff --git a/apps/workbench/app/views/users/_show_admin.html.erb b/apps/workbench/app/views/users/_show_admin.html.erb
new file mode 100644 (file)
index 0000000..6e60b5d
--- /dev/null
@@ -0,0 +1,7 @@
+<p>As an admin, you can log in as this user. When you&rsquo;ve
+finished, you will need to log out and log in again with your own
+account.</p>
+
+<blockquote>
+<%= button_to "Log in as #{@object.full_name}", sudo_user_url(id: @object.uuid), class: 'btn btn-primary' %>
+</blockquote>
diff --git a/apps/workbench/config/application.default.yml b/apps/workbench/config/application.default.yml
new file mode 100644 (file)
index 0000000..1392552
--- /dev/null
@@ -0,0 +1,66 @@
+# Do not use this file for site configuration. Create config.yml
+# instead (see application.yml.example).
+
+development:
+  cache_classes: false
+  whiny_nils: true
+  consider_all_requests_local: true
+  action_controller.perform_caching: false
+  action_mailer.raise_delivery_errors: false
+  active_support.deprecation: :log
+  action_dispatch.best_standards_support: :builtin
+  active_record.mass_assignment_sanitizer: :strict
+  active_record.auto_explain_threshold_in_seconds: 0.5
+  assets.compress: false
+  assets.debug: true
+  profiling_enabled: true
+  site_name: Workbench:dev
+
+production:
+  force_ssl: true
+  cache_classes: true
+  consider_all_requests_local: false
+  action_controller.perform_caching: true
+  serve_static_assets: false
+  assets.compress: true
+  assets.compile: false
+  assets.digest: true
+  i18n.fallbacks: true
+  active_support.deprecation: :notify
+  profiling_enabled: false
+
+  arvados_insecure_https: false
+
+  data_import_dir: /data/arvados-workbench-upload/data
+  data_export_dir: /data/arvados-workbench-download/data
+
+  site_name: Arvados Workbench
+
+test:
+  cache_classes: true
+  serve_static_assets: true
+  static_cache_control: public, max-age=3600
+  whiny_nils: true
+  consider_all_requests_local: true
+  action_controller.perform_caching: false
+  action_dispatch.show_exceptions: false
+  action_controller.allow_forgery_protection: false
+  action_mailer.delivery_method: :test
+  active_record.mass_assignment_sanitizer: :strict
+  active_support.deprecation: :stderr
+  profiling_enabled: false
+  secret_token: <%= rand(2**256).to_s(36) %>
+
+  site_name: Workbench:test
+
+common:
+  data_import_dir: /tmp/arvados-workbench-upload
+  data_export_dir: /tmp/arvados-workbench-download
+  arvados_login_base: https://arvados.local/login
+  arvados_v1_base: https://arvados.local/arvados/v1
+  arvados_insecure_https: true
+  activation_contact_link: mailto:info@arvados.org
+  arvados_docsite: http://doc.arvados.org
+  arvados_theme: default
+  show_user_agreement_inline: false
+  secret_token: ~
diff --git a/apps/workbench/config/application.yml.example b/apps/workbench/config/application.yml.example
new file mode 100644 (file)
index 0000000..395f1a9
--- /dev/null
@@ -0,0 +1,20 @@
+# Copy this file to application.yml and edit to suit.
+#
+# Consult application.default.yml for the full list of configuration
+# settings.
+#
+# The order of precedence is:
+# 1. config/environments/{RAILS_ENV}.rb (deprecated)
+# 2. Section in application.yml corresponding to RAILS_ENV (e.g., development)
+# 3. Section in application.yml called "common"
+# 4. Section in application.default.yml corresponding to RAILS_ENV
+# 5. Section in application.default.yml called "common"
+
+common:
+  # At minimum, you need a nice long randomly generated secret_token here.
+  secret_token: ~
+
+  # You probably also want to point to your API server.
+  arvados_login_base: https://arvados.local:3000/login
+  arvados_v1_base: https://arvados.local:3000/arvados/v1
+  arvados_insecure_https: true
diff --git a/apps/workbench/config/initializers/zz_load_config.rb b/apps/workbench/config/initializers/zz_load_config.rb
new file mode 100644 (file)
index 0000000..43711fc
--- /dev/null
@@ -0,0 +1,46 @@
+$application_config = {}
+
+%w(application.default application).each do |cfgfile|
+  path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
+  if File.exists? path
+    yaml = ERB.new(IO.read path).result(binding)
+    confs = YAML.load(yaml)
+    $application_config.merge!(confs['common'] || {})
+    $application_config.merge!(confs[::Rails.env.to_s] || {})
+  end
+end
+
+ArvadosWorkbench::Application.configure do
+  nils = []
+  $application_config.each do |k, v|
+    # "foo.bar: baz" --> { config.foo.bar = baz }
+    cfg = config
+    ks = k.split '.'
+    k = ks.pop
+    ks.each do |kk|
+      cfg = cfg.send(kk)
+    end
+    if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
+      # Config must have been set already in environments/*.rb.
+      #
+      # After config files have been migrated, this mechanism should
+      # be deprecated, then removed.
+    elsif v.nil?
+      # Config variables are not allowed to be nil. Make a "naughty"
+      # list, and present it below.
+      nils << k
+    else
+      cfg.send "#{k}=", v
+    end
+  end
+  if !nils.empty?
+    raise <<EOS
+Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
+
+The following configuration settings must be specified in
+config/application.yml:
+* #{nils.join "\n* "}
+
+EOS
+  end
+end
index 5330a9148a2f8574c0d410e8ff83acb67eaa4911..527d6efef5e4a0ed9e058b361fbd3e567a9f88bd 100644 (file)
@@ -19,6 +19,7 @@ ArvadosWorkbench::Application.routes.draw do
   resources :users do
     get 'home', :on => :member
     get 'welcome', :on => :collection
+    post 'sudo', :on => :member
   end
   resources :logs
   resources :factory_jobs
diff --git a/apps/workbench/lib/tasks/config_check.rake b/apps/workbench/lib/tasks/config_check.rake
new file mode 100644 (file)
index 0000000..c9f12fc
--- /dev/null
@@ -0,0 +1,8 @@
+namespace :config do
+  desc 'Ensure site configuration has all required settings'
+  task check: :environment do
+    $application_config.sort.each do |k, v|
+      $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+    end
+  end
+end
index 1dec2bb454e8b0ce0778d0c50b668426bc5373f9..f8c3ee545fa0d7d3802b3a7925d8dc8411754d2a 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_client_authorization@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/api_client_authorizations@
 
 h2. Creation
 
index 7d8bc20f4b534a2d72b08d374192a1ff832931b0..3606a26c82762c9ddf44fefe7d69b744131c8a0e 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/authorized_key@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/authorized_keys@
 
 h2. Creation
 
index 403d8d5da033567d6d18c5505a0e68a238ecafa0..5bb95e1907e09c1b75c018db11e89f4ecf06ae1c 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commits@
 
 h2. Creation
 
index ab4dd5b4a90a99cdbb9d6d5b50bac6802d22d749..904eec7cf4f696d38e5149887712537f959811df 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit_ancestor@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/commit_ancestors@
 
 h2. Creation
 
index 71a006c1867c8a7a561d48b79665857bbffdeaff..7fe3f11c265895b270996d0d98ad0f533832f1f5 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/group@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/groups@
 
 h2. Creation
 
index 837fc4209f9c80a73440ab8df0efd417294ce016..82c6c30e0c9e85cde3bb065d8be3051024d544ae 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/human@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/humans@
 
 h2. Creation
 
index edac4d5e12625e2b39f119c09dd7b06ee349e117..cfaedc5d10ae3f75d414d9ef21dc1d4e3e627337 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/keep_disk@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/keep_disks@
 
 h2. Creation
 
index 533ca5c2caeb1a9e3c0ba26248b2389cf8d22abd..0435b8c0bb040c9d5f504628099810e687059b94 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/log@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/logs@
 
 h2. Creation
 
index 827ce18bf676cb9f3ac19d7083feee8f9174b474..a05bd3a4ea511de85c33e5deeab86f08b7093878 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/node@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/nodes@
 
 h2. Creation
 
index 7fd62f06e09db7a533c2b18d51a2b691dca7f856..779c226ef8bfc3ce80bb5f2daa62364f261733ac 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_instance@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_instances@
 
 h2. Creation
 
index 75b99815075c51f7b8f2c7cb7e433ba37f6876ac..7fc4959184e2dabe517da6bc6803e26409dee72b 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_template@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/pipeline_templates@
 
 h2. Creation
 
index e21b7432cdc2d554f0869b559dcf9982739c36a6..c76ab82ee9369a8d888e64a41ac9ff0ceea8bacf 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/repository@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/repositories@
 
 h2. Creation
 
index 7c0eff94fc56b146ffdc1389a0c84b8b36a68f91..ab19bc0e5ab2f598471d63af0041674fb2396396 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/specimen@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/specimens@
 
 h2. Creation
 
index 8f077ef70ab15e34ccf383743f9228a2cb742142..e7cb741948e210e23879f010429abeca8c06a19b 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/trait@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/traits@
 
 h2. Creation
 
index cd7e64b9721493658661cbca60f2e655c42b533e..76d49787e1dee4cef62366dcda57b18468da9102 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/user@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/users@
 
 h2. Creation
 
index ba5b3a782428a2f18290316322ffd2a1e1df5854..1e1de5fd452a84024008767d63ab0701e5d3d8b3 100644 (file)
@@ -14,7 +14,7 @@ h2. Methods
 
 See "REST methods for working with Arvados resources":/api/methods.html
 
-API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/virtual_machine@
+API endpoint base: @https://{{ site.arvados_api_host }}/arvados/v1/virtual_machines@
 
 h2. Creation
 
index a16e50f94a97e07fb0ad4db70f0a88c29dc30e9c..582feb95b1543c24ea9541c454bd142f27e18a8b 100644 (file)
@@ -5,18 +5,20 @@ title: Install the API server
 navorder: 1
 ...
 
-{% include 'alert_stub' %}
-
 # API server setup
 
-## Prerequisites
+## Prerequisites:
 
-1. A GNU/linux (virtual) machine
+1. A GNU/Linux (virtual) machine
 2. A domain name for your api server
+3. Ruby >= 2.0.0
+4. Bundler
 
 ## Download the source tree
 
-Please follow the instructions on the [Download page](https://arvados.org/projects/arvados/wiki/Download) in the wiki.
+    git clone https://github.com/curoverse/arvados.git
+
+See also: [Downloading the source code](https://arvados.org/projects/arvados/wiki/Download) on the Arvados wiki.
 
 ## Configure the API server
 
@@ -27,61 +29,58 @@ First install the gems:
 
 Next, configure the database:
 
-    cp config/database.yml.sample config/database.yml
+    cp -i config/database.yml.sample config/database.yml
 
-Edit database.yml to your liking and make sure the database and db user exist.
+Edit `database.yml` to your liking and make sure the database and db user exist.
 Then set up the database:
  
     RAILS_ENV=production rake db:setup
 
 Then set up omniauth:
 
-    cp config/initializers/omniauth.rb.example config/initializers/omniauth.rb
+    cp -i config/initializers/omniauth.rb.example config/initializers/omniauth.rb
 
-Edit config/initializers/omniauth.rb. Choose an *APP_SECRET* and *APP_ID*. Also set
-*CUSTOM_PROVIDER_URL*.
+Edit `config/initializers/omniauth.rb`. Choose an *APP_SECRET* and
+*APP_ID*. Also set *CUSTOM_PROVIDER_URL*.
 
 Make sure your Omniauth provider knows about your *APP_ID* and *APP_SECRET*
 combination.
 
-You also need to update config/initializers/secret_token.rb. Generate a new secret with
+You also need to update `config/initializers/secret_token.rb`.
+Generate a new secret with
 
     rake secret
 
-and put it in config/initializers/secret_token.rb:
+and put it in `config/initializers/secret_token.rb`:
 
     Server::Application.config.secret_token = 'your-new-secret-here'
 
-Finally, edit the main configuration:
+Edit the main configuration:
 
-    cp config/environments/production.rb.example config/environments/production.rb
+    cp -i config/config.yml.example config/config.yml
 
-First, you want to make sure that 
+First, you want to make sure that
 
-    config.uuid_prefix
+    uuid_prefix
 
-is set to a unique 5-digit hex string. You can replace the 'cfi-aws-0' string
-with a string of your choice to make that happen.
+is set to a unique 5-character alphanumeric string. An example is
+given that generates a 5-character string based on a hash of your
+hostname.
 
-The *config.uuid_prefix* string is a unique identifier for your API server. It
-also serves as the first part of the hostname for your API server, for instance
+The `uuid_prefix` is a unique identifier for your API server. It also
+serves as the first part of the hostname for your API server, for
+instance
 
     {{ site.arvados_api_host }}
 
-You should use your own domain instead of arvadosapi.com
-
-Second, unless you are running on AWS, you will want to change the definition of
-
-    config.compute_node_nameservers
+For a development site, use your own domain instead of arvadosapi.com.
 
-If you know your nameservers and they are fixed, you can hardcode them, and
-make sure to remove the code that tries to look them up from the AWS metadata:
+You will also want to change `compute_node_nameservers` to suit your
+environment.
 
-    config.compute_node_nameservers = ['1.2.3.4','2.3.4.5','3.4.5.6']
-    #require 'net/http'
-    #config.compute_node_nameservers = ['local', 'public'].collect do |iface|
-    #  Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
-    #end << '172.16.0.23'
+Consult `config.defaults.yml` for a full list of configuration
+options. Always put your local configuration in `config.yml` instead
+of editing `config.defaults.yml`.
 
 ## Apache/Passenger
 
index b48b6df009dbe26ca970c381b0202472eb26e6fe..ff29919f1bf3c6a0e7e60245333ad454164f2df5 100644 (file)
@@ -84,12 +84,15 @@ def normalize(collection):
 
 class CollectionReader(object):
     def __init__(self, manifest_locator_or_text):
-        if re.search(r'^[a-f0-9]{32}\+\d+(\+\S)*$', manifest_locator_or_text):
+        if re.search(r'^[a-f0-9]{32}(\+\d+)?(\+\S+)*$', manifest_locator_or_text):
             self._manifest_locator = manifest_locator_or_text
             self._manifest_text = None
-        else:
+        elif re.search(r'^\S+( [a-f0-9]{32,}(\+\S+)*)*( \d+:\d+:\S+)+\n', manifest_locator_or_text):
             self._manifest_text = manifest_locator_or_text
             self._manifest_locator = None
+        else:
+            raise errors.ArgumentError(
+                "Argument to CollectionReader must be a manifest or a collection UUID")
         self._streams = None
 
     def __enter__(self):
index 5ea54befdebcc735dabb3b13688f50743474f31e..e4c69a3c83dff24ebc70f1ce0212931b4b77ba06 100644 (file)
@@ -1,5 +1,7 @@
 # errors.py - Arvados-specific exceptions.
 
+class ArgumentError(Exception):
+    pass
 class SyntaxError(Exception):
     pass
 class AssertionError(Exception):
index 3dfc72f65b66bcc25482e966edda8d85795daa12..7df620d977d54954fbd9b0b7d943549934b467cd 100644 (file)
@@ -42,7 +42,7 @@ class LocalCollectionReaderTest(unittest.TestCase):
         os.environ['KEEP_LOCAL_STORE'] = '/tmp'
         LocalCollectionWriterTest().runTest()
     def runTest(self):
-        cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114')
+        cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114+Xzizzle')
         got = []
         for s in cr.all_streams():
             for f in s.all_files():
index 80ba00019016bce1d9ca97324bf740215b60d450..6ddf5231ced091461771a94999948f521136ad55 100644 (file)
@@ -18,6 +18,7 @@
 /config/api.clinicalfuture.com.*
 /config/database.yml
 /config/initializers/omniauth.rb
+/config/application.yml
 
 # asset cache
 /public/assets/
index 10a009807cf171001842e1e84f077957c0cf9516..8fd915ddfbf48d8b3a336d47e58257147f3c6899 100644 (file)
@@ -28,6 +28,7 @@ class Arvados::V1::ApiClientAuthorizationsController < ApplicationController
       resource_attrs[:user_id] =
         User.where(uuid: resource_attrs.delete(:owner_uuid)).first.andand.id
     end
+    resource_attrs[:api_client_id] = Thread.current[:api_client].id
     super
   end
 
index 8ee14b793667f86e44a6fbb311fbee402689a14b..c89efdf404abb3a0f7f9a374562851b57abe372d 100644 (file)
@@ -136,7 +136,7 @@ class ArvadosModel < ActiveRecord::Base
 
   def update_modified_by_fields
     self.created_at ||= Time.now
-    self.owner_uuid ||= current_default_owner
+    self.owner_uuid ||= current_default_owner if self.respond_to? :owner_uuid=
     self.modified_at = Time.now
     self.modified_by_user_uuid = current_user ? current_user.uuid : nil
     self.modified_by_client_uuid = current_api_client ? current_api_client.uuid : nil
diff --git a/services/api/app/models/blob.rb b/services/api/app/models/blob.rb
new file mode 100644 (file)
index 0000000..11fab9f
--- /dev/null
@@ -0,0 +1,96 @@
+class Blob
+
+  # In order to get a Blob from Keep, you have to prove either
+  # [a] you have recently written it to Keep yourself, or
+  # [b] apiserver has recently decided that you should be able to read it
+  #
+  # To ensure that the requestor of a blob is authorized to read it,
+  # Keep requires clients to timestamp the blob locator with an expiry
+  # time, and to sign the timestamped locator with their API token.
+  #
+  # A signed blob locator has the form:
+  #     locator_hash +A blob_signature @ timestamp
+  # where the timestamp is a Unix time expressed as a hexadecimal value,
+  # and the blob_signature is the signed locator_hash + API token + timestamp.
+  # 
+  class InvalidSignatureError < StandardError
+  end
+
+  # Blob.sign_locator: return a signed and timestamped blob locator.
+  #
+  # The 'opts' argument should include:
+  #   [required] :key       - the Arvados server-side blobstore key
+  #   [required] :api_token - user's API token
+  #   [optional] :ttl       - number of seconds before this request expires
+  #
+  def self.sign_locator blob_locator, opts
+    # We only use the hash portion for signatures.
+    blob_hash = blob_locator.split('+').first
+
+    # Generate an expiry timestamp (seconds since epoch, base 16)
+    timestamp = (Time.now.to_i + (opts[:ttl] || 600)).to_s(16)
+    # => "53163cb4"
+
+    # Generate a signature.
+    signature =
+      generate_signature opts[:key], blob_hash, opts[:api_token], timestamp
+
+    blob_locator + '+A' + signature + '@' + timestamp
+  end
+
+  # Blob.verify_signature
+  #   Safely verify the signature on a blob locator.
+  #   Return value: true if the locator has a valid signature, false otherwise
+  #   Arguments: signed_blob_locator, opts
+  #
+  def self.verify_signature *args
+    begin
+      self.verify_signature! *args
+      true
+    rescue Blob::InvalidSignatureError
+      false
+    end
+  end
+
+  # Blob.verify_signature!
+  #   Verify the signature on a blob locator.
+  #   Return value: true if the locator has a valid signature
+  #   Arguments: signed_blob_locator, opts
+  #   Exceptions:
+  #     Blob::InvalidSignatureError if the blob locator does not include a
+  #     valid signature
+  #
+  def self.verify_signature! signed_blob_locator, opts
+    blob_hash = signed_blob_locator.split('+').first
+    given_signature, timestamp = signed_blob_locator.
+      split('+A').last.
+      split('+').first.
+      split('@')
+
+    if !timestamp
+      raise Blob::InvalidSignatureError.new 'No signature provided.'
+    end
+    if !timestamp.match /^[\da-f]+$/
+      raise Blob::InvalidSignatureError.new 'Timestamp is not a base16 number.'
+    end
+    if timestamp.to_i(16) < Time.now.to_i
+      raise Blob::InvalidSignatureError.new 'Signature expiry time has passed.'
+    end
+
+    my_signature =
+      generate_signature opts[:key], blob_hash, opts[:api_token], timestamp
+
+    if my_signature != given_signature
+      raise Blob::InvalidSignatureError.new 'Signature is invalid.'
+    end
+
+    true
+  end
+
+  def self.generate_signature key, blob_hash, api_token, timestamp
+    OpenSSL::HMAC.hexdigest('sha1', key,
+                            [blob_hash,
+                             api_token,
+                             timestamp].join('@'))
+  end
+end
index 459535b52df9450741b589f9a84eb94a738dadfc..805e1ccd41cabd1a681e901b4cdb38f7d375067a 100644 (file)
@@ -8,13 +8,7 @@ class Node < ArvadosModel
 
   MAX_SLOTS = 64
 
-  @@confdir = if Rails.configuration.respond_to? :dnsmasq_conf_dir
-                Rails.configuration.dnsmasq_conf_dir
-              elsif File.exists? '/etc/dnsmasq.d/.'
-                '/etc/dnsmasq.d'
-              else
-                nil
-              end
+  @@confdir = Rails.configuration.dnsmasq_conf_dir
   @@domain = Rails.configuration.compute_node_domain rescue `hostname --domain`.strip
   @@nameservers = Rails.configuration.compute_node_nameservers
 
@@ -127,8 +121,8 @@ class Node < ArvadosModel
   def start!(ping_url_method)
     ensure_permission_to_update
     ping_url = ping_url_method.call({ uuid: self.uuid, ping_secret: self.info[:ping_secret] })
-    if (Rails.configuration.compute_node_ec2run_args rescue false) and
-       (Rails.configuration.compute_node_ami rescue false)
+    if (Rails.configuration.compute_node_ec2run_args and
+        Rails.configuration.compute_node_ami)
       ec2_args = ["--user-data '#{ping_url}'",
                   "-t c1.xlarge -n 1",
                   Rails.configuration.compute_node_ec2run_args,
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
new file mode 100644 (file)
index 0000000..36eedb4
--- /dev/null
@@ -0,0 +1,86 @@
+# Do not use this file for site configuration. Create application.yml
+# instead (see application.yml.example).
+
+development:
+  force_ssl: false
+  cache_classes: false
+  whiny_nils: true
+  consider_all_requests_local: true
+  action_controller.perform_caching: false
+  action_mailer.raise_delivery_errors: false
+  action_mailer.perform_deliveries: false
+  active_support.deprecation: :log
+  action_dispatch.best_standards_support: :builtin
+  active_record.mass_assignment_sanitizer: :strict
+  active_record.auto_explain_threshold_in_seconds: 0.5
+  assets.compress: false
+  assets.debug: true
+
+production:
+  force_ssl: true
+  cache_classes: true
+  consider_all_requests_local: false
+  action_controller.perform_caching: true
+  serve_static_assets: false
+  assets.compress: true
+  assets.compile: false
+  assets.digest: true
+
+test:
+  force_ssl: false
+  cache_classes: true
+  serve_static_assets: true
+  static_cache_control: public, max-age=3600
+  whiny_nils: true
+  consider_all_requests_local: true
+  action_controller.perform_caching: false
+  action_dispatch.show_exceptions: false
+  action_controller.allow_forgery_protection: false
+  action_mailer.delivery_method: :test
+  active_support.deprecation: :stderr
+  active_record.mass_assignment_sanitizer: :strict
+
+common:
+  secret_token: ~
+  uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
+
+  git_repositories_dir: /var/cache/git
+
+  # :none or :slurm_immediate
+  crunch_job_wrapper: :none
+
+  # username, or false = do not set uid when running jobs.
+  crunch_job_user: crunch
+
+  # The web service must be able to create/write this file, and
+  # crunch-job must be able to stat() it.
+  crunch_refresh_trigger: /tmp/crunch_refresh_trigger
+
+  # Path to /etc/dnsmasq.d, or false = do not update dnsmasq data.
+  dnsmasq_conf_dir: false
+
+  # Set to AMI id (ami-123456) to auto-start nodes. See app/models/node.rb
+  compute_node_ami: false
+  compute_node_ec2run_args: -g arvados-compute
+  compute_node_spot_bid: 0.11
+
+  compute_node_domain: <%= `hostname`.split('.')[1..-1].join('.').strip %>
+  compute_node_nameservers:
+    - 192.168.1.1
+  compute_node_ec2_tag_enable: false
+
+  accept_api_token: {}
+
+  new_users_are_active: false
+  admin_notifier_email_from: arvados@example.com
+  email_subject_prefix: "[ARVADOS] "
+
+  # Visitors to the API server will be redirected to the workbench
+  workbench_address: https://workbench.local:3001/
+
+  # The e-mail address of the user you would like to become marked as an admin
+  # user on their first login.
+  # In the default configuration, authentication happens through the Arvados SSO
+  # server, which uses openid against Google's servers, so in that case this
+  # should be an address associated with a Google account.
+  auto_admin_user: ~
diff --git a/services/api/config/application.yml.example b/services/api/config/application.yml.example
new file mode 100644 (file)
index 0000000..dcfcb42
--- /dev/null
@@ -0,0 +1,41 @@
+# Copy this file to application.yml and edit to suit.
+#
+# Consult application.default.yml for the full list of configuration
+# settings.
+#
+# The order of precedence is:
+# 1. config/environments/{RAILS_ENV}.rb (deprecated)
+# 2. Section in application.yml corresponding to RAILS_ENV (e.g., development)
+# 3. Section in application.yml called "common"
+# 4. Section in application.default.yml corresponding to RAILS_ENV
+# 5. Section in application.default.yml called "common"
+
+development:
+
+production:
+  # At minimum, you need a nice long randomly generated secret_token here.
+  secret_token: ~
+
+  uuid_prefix: bogus
+
+  # This is suitable for AWS; see common section below for a static example.
+  compute_node_nameservers: <%=
+    require 'net/http'
+    ['local', 'public'].collect do |iface|
+      Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
+    end << '172.16.0.23'
+  %>
+  # You must customize these. See config.defaults.yml for information.
+  compute_node_ami: ~
+  compute_node_ec2_tag_enable: ~
+  compute_node_domain: ~
+  compute_node_spot_bid: ~
+
+test:
+  uuid_prefix: zzzzz
+
+common:
+  secret_token: ~
+  compute_node_nameservers:
+    - 192.168.0.1
+    - 172.16.0.1
index 62edd8431e74c86c7598c733c8bd1067c79ea7f3..8f54e6630987377588d8ce0e00a2e59a3434b351 100644 (file)
@@ -1,5 +1,5 @@
 development:
-  adapter: mysql
+  adapter: postgresql
   encoding: utf8
   database: arvados_development
   username: arvados
@@ -7,7 +7,7 @@ development:
   host: localhost
 
 test:
-  adapter: mysql
+  adapter: postgresql
   encoding: utf8
   database: arvados_test
   username: arvados
@@ -15,7 +15,7 @@ test:
   host: localhost
 
 production:
-  adapter: mysql
+  adapter: postgresql
   encoding: utf8
   database: arvados_production
   username: arvados
index 1782734f83f63d6a25575a119d0bad7142f2270e..10608c15e356fdbede112a7f523db26dedd8db09 100644 (file)
@@ -76,4 +76,11 @@ Server::Application.configure do
 
   # Visitors to the API server will be redirected to the workbench
   config.workbench_address = "http://localhost:3000/"
+
+  # The e-mail address of the user you would like to become marked as an admin
+  # user on their first login.
+  # In the default configuration, authentication happens through the Arvados SSO
+  # server, which uses openid against Google's servers, so in that case this
+  # should be an address associated with a Google account.
+  config.auto_admin_user = ''
 end
diff --git a/services/api/config/initializers/zz_load_config.rb b/services/api/config/initializers/zz_load_config.rb
new file mode 100644 (file)
index 0000000..3399fd9
--- /dev/null
@@ -0,0 +1,46 @@
+$application_config = {}
+
+%w(application.default application).each do |cfgfile|
+  path = "#{::Rails.root.to_s}/config/#{cfgfile}.yml"
+  if File.exists? path
+    yaml = ERB.new(IO.read path).result(binding)
+    confs = YAML.load(yaml)
+    $application_config.merge!(confs['common'] || {})
+    $application_config.merge!(confs[::Rails.env.to_s] || {})
+  end
+end
+
+Server::Application.configure do
+  nils = []
+  $application_config.each do |k, v|
+    # "foo.bar: baz" --> { config.foo.bar = baz }
+    cfg = config
+    ks = k.split '.'
+    k = ks.pop
+    ks.each do |kk|
+      cfg = cfg.send(kk)
+    end
+    if cfg.respond_to?(k.to_sym) and !cfg.send(k).nil?
+      # Config must have been set already in environments/*.rb.
+      #
+      # After config files have been migrated, this mechanism should
+      # be deprecated, then removed.
+    elsif v.nil?
+      # Config variables are not allowed to be nil. Make a "naughty"
+      # list, and present it below.
+      nils << k
+    else
+      cfg.send "#{k}=", v
+    end
+  end
+  if !nils.empty?
+    raise <<EOS
+Refusing to start in #{::Rails.env.to_s} mode with missing configuration.
+
+The following configuration settings must be specified in
+config/application.yml:
+* #{nils.join "\n* "}
+
+EOS
+  end
+end
diff --git a/services/api/lib/tasks/config_check.rake b/services/api/lib/tasks/config_check.rake
new file mode 100644 (file)
index 0000000..c9f12fc
--- /dev/null
@@ -0,0 +1,8 @@
+namespace :config do
+  desc 'Ensure site configuration has all required settings'
+  task check: :environment do
+    $application_config.sort.each do |k, v|
+      $stderr.puts "%-32s %s" % [k, eval("Rails.configuration.#{k}")]
+    end
+  end
+end
index 5c3c0ddfea47b3678956e76d90d72ab5ffb1bca7..fef4b5bb21eea7449061e93d84bd9a718d29f64e 100644 (file)
@@ -8,4 +8,40 @@ class ApiClientAuthorizationsApiTest < ActionDispatch::IntegrationTest
     assert_response :success
   end
 
+  test "create token for different user" do
+    post "/arvados/v1/api_client_authorizations", {
+      :format => :json,
+      :api_client_authorization => {
+        :owner_uuid => users(:spectator).uuid
+      }
+    }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin_trustedclient).api_token}"}
+    assert_response :success
+
+    get "/arvados/v1/users/current", {
+      :format => :json
+    }, {'HTTP_AUTHORIZATION' => "OAuth2 #{jresponse['api_token']}"}
+    @jresponse = nil
+    assert_equal users(:spectator).uuid, jresponse['uuid']
+  end
+
+  test "refuse to create token for different user if not trusted client" do
+    post "/arvados/v1/api_client_authorizations", {
+      :format => :json,
+      :api_client_authorization => {
+        :owner_uuid => users(:spectator).uuid
+      }
+    }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:admin).api_token}"}
+    assert_response 403
+  end
+
+  test "refuse to create token for different user if not admin" do
+    post "/arvados/v1/api_client_authorizations", {
+      :format => :json,
+      :api_client_authorization => {
+        :owner_uuid => users(:spectator).uuid
+      }
+    }, {'HTTP_AUTHORIZATION' => "OAuth2 #{api_client_authorizations(:active_trustedclient).api_token}"}
+    assert_response 403
+  end
+
 end
diff --git a/services/api/test/unit/blob_test.rb b/services/api/test/unit/blob_test.rb
new file mode 100644 (file)
index 0000000..ec6e67a
--- /dev/null
@@ -0,0 +1,94 @@
+require 'test_helper'
+
+class BlobTest < ActiveSupport::TestCase
+  @@api_token = rand(2**512).to_s(36)[0..49]
+  @@key = rand(2**2048).to_s(36)
+  @@blob_data = 'foo'
+  @@blob_locator = Digest::MD5.hexdigest(@@blob_data) +
+    '+' + @@blob_data.size.to_s
+
+  test 'correct' do
+    signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key
+    assert_equal true, Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+  end
+
+  test 'expired' do
+    signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key, ttl: -1
+    assert_raise Blob::InvalidSignatureError do
+      Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+    end
+  end
+
+  test 'expired, but no raise' do
+    signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key, ttl: -1
+    assert_equal false, Blob.verify_signature(signed,
+                                              api_token: @@api_token,
+                                              key: @@key)
+  end
+
+  test 'bogus, wrong block hash' do
+    signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key
+    assert_raise Blob::InvalidSignatureError do
+      Blob.verify_signature!(signed.sub('acbd','abcd'), api_token: @@api_token, key: @@key)
+    end
+  end
+
+  test 'bogus, expired' do
+    signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@531641bf'
+    assert_raises Blob::InvalidSignatureError do
+      Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+    end
+  end
+
+  test 'bogus, wrong key' do
+    signed = Blob.sign_locator(@@blob_locator,
+                               api_token: @@api_token,
+                               key: (@@key+'x'))
+    assert_raise Blob::InvalidSignatureError do
+      Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+    end
+  end
+
+  test 'bogus, wrong api token' do
+    signed = Blob.sign_locator(@@blob_locator,
+                               api_token: @@api_token.reverse,
+                               key: @@key)
+    assert_raise Blob::InvalidSignatureError do
+      Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+    end
+  end
+
+  test 'bogus, signature format 1' do
+    signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@'
+    assert_raise Blob::InvalidSignatureError do
+      Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+    end
+  end
+
+  test 'bogus, signature format 2' do
+    signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+A@531641bf'
+    assert_raise Blob::InvalidSignatureError do
+      Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+    end
+  end
+
+  test 'bogus, signature format 3' do
+    signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Axyzzy@531641bf'
+    assert_raise Blob::InvalidSignatureError do
+      Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+    end
+  end
+
+  test 'bogus, timestamp format' do
+    signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@xyzzy'
+    assert_raise Blob::InvalidSignatureError do
+      Blob.verify_signature!(signed, api_token: @@api_token, key: @@key)
+    end
+  end
+
+  test 'no signature at all' do
+    assert_raise Blob::InvalidSignatureError do
+      Blob.verify_signature!(@@blob_locator, api_token: @@api_token, key: @@key)
+    end
+  end
+end