From: Tim Pierce Date: Mon, 10 Mar 2014 14:49:41 +0000 (-0400) Subject: Merge branch 'master' into 2221-complete-docker X-Git-Tag: 1.1.0~2690^2~64^2~31 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/e7b601c25f6ac88e7adaf219f296c448d29c3904?hp=12af96fba8ef1c218cb3cfb04c41c5507b394717 Merge branch 'master' into 2221-complete-docker --- diff --git a/apps/workbench/app/views/users/_tables.html.erb b/apps/workbench/app/views/users/_tables.html.erb index 320f32c571..1592632f4a 100644 --- a/apps/workbench/app/views/users/_tables.html.erb +++ b/apps/workbench/app/views/users/_tables.html.erb @@ -27,7 +27,7 @@ - <%= link_to j.script[0..31], job_path(j.uuid) %> + <%= link_to((j.script.andand[0..31] || j.uuid), job_path(j.uuid)) %> diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py index b48b6df009..ff29919f1b 100644 --- a/sdk/python/arvados/collection.py +++ b/sdk/python/arvados/collection.py @@ -84,12 +84,15 @@ def normalize(collection): class CollectionReader(object): def __init__(self, manifest_locator_or_text): - if re.search(r'^[a-f0-9]{32}\+\d+(\+\S)*$', manifest_locator_or_text): + if re.search(r'^[a-f0-9]{32}(\+\d+)?(\+\S+)*$', manifest_locator_or_text): self._manifest_locator = manifest_locator_or_text self._manifest_text = None - else: + elif re.search(r'^\S+( [a-f0-9]{32,}(\+\S+)*)*( \d+:\d+:\S+)+\n', manifest_locator_or_text): self._manifest_text = manifest_locator_or_text self._manifest_locator = None + else: + raise errors.ArgumentError( + "Argument to CollectionReader must be a manifest or a collection UUID") self._streams = None def __enter__(self): diff --git a/sdk/python/arvados/errors.py b/sdk/python/arvados/errors.py index 5ea54befde..e4c69a3c83 100644 --- a/sdk/python/arvados/errors.py +++ b/sdk/python/arvados/errors.py @@ -1,5 +1,7 @@ # errors.py - Arvados-specific exceptions. +class ArgumentError(Exception): + pass class SyntaxError(Exception): pass class AssertionError(Exception): diff --git a/sdk/python/test_collections.py b/sdk/python/test_collections.py index 3dfc72f65b..7df620d977 100644 --- a/sdk/python/test_collections.py +++ b/sdk/python/test_collections.py @@ -42,7 +42,7 @@ class LocalCollectionReaderTest(unittest.TestCase): os.environ['KEEP_LOCAL_STORE'] = '/tmp' LocalCollectionWriterTest().runTest() def runTest(self): - cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114') + cr = arvados.CollectionReader('d6c3b8e571f1b81ebb150a45ed06c884+114+Xzizzle') got = [] for s in cr.all_streams(): for f in s.all_files(): diff --git a/services/api/app/models/blob.rb b/services/api/app/models/blob.rb new file mode 100644 index 0000000000..11fab9fb59 --- /dev/null +++ b/services/api/app/models/blob.rb @@ -0,0 +1,96 @@ +class Blob + + # In order to get a Blob from Keep, you have to prove either + # [a] you have recently written it to Keep yourself, or + # [b] apiserver has recently decided that you should be able to read it + # + # To ensure that the requestor of a blob is authorized to read it, + # Keep requires clients to timestamp the blob locator with an expiry + # time, and to sign the timestamped locator with their API token. + # + # A signed blob locator has the form: + # locator_hash +A blob_signature @ timestamp + # where the timestamp is a Unix time expressed as a hexadecimal value, + # and the blob_signature is the signed locator_hash + API token + timestamp. + # + class InvalidSignatureError < StandardError + end + + # Blob.sign_locator: return a signed and timestamped blob locator. + # + # The 'opts' argument should include: + # [required] :key - the Arvados server-side blobstore key + # [required] :api_token - user's API token + # [optional] :ttl - number of seconds before this request expires + # + def self.sign_locator blob_locator, opts + # We only use the hash portion for signatures. + blob_hash = blob_locator.split('+').first + + # Generate an expiry timestamp (seconds since epoch, base 16) + timestamp = (Time.now.to_i + (opts[:ttl] || 600)).to_s(16) + # => "53163cb4" + + # Generate a signature. + signature = + generate_signature opts[:key], blob_hash, opts[:api_token], timestamp + + blob_locator + '+A' + signature + '@' + timestamp + end + + # Blob.verify_signature + # Safely verify the signature on a blob locator. + # Return value: true if the locator has a valid signature, false otherwise + # Arguments: signed_blob_locator, opts + # + def self.verify_signature *args + begin + self.verify_signature! *args + true + rescue Blob::InvalidSignatureError + false + end + end + + # Blob.verify_signature! + # Verify the signature on a blob locator. + # Return value: true if the locator has a valid signature + # Arguments: signed_blob_locator, opts + # Exceptions: + # Blob::InvalidSignatureError if the blob locator does not include a + # valid signature + # + def self.verify_signature! signed_blob_locator, opts + blob_hash = signed_blob_locator.split('+').first + given_signature, timestamp = signed_blob_locator. + split('+A').last. + split('+').first. + split('@') + + if !timestamp + raise Blob::InvalidSignatureError.new 'No signature provided.' + end + if !timestamp.match /^[\da-f]+$/ + raise Blob::InvalidSignatureError.new 'Timestamp is not a base16 number.' + end + if timestamp.to_i(16) < Time.now.to_i + raise Blob::InvalidSignatureError.new 'Signature expiry time has passed.' + end + + my_signature = + generate_signature opts[:key], blob_hash, opts[:api_token], timestamp + + if my_signature != given_signature + raise Blob::InvalidSignatureError.new 'Signature is invalid.' + end + + true + end + + def self.generate_signature key, blob_hash, api_token, timestamp + OpenSSL::HMAC.hexdigest('sha1', key, + [blob_hash, + api_token, + timestamp].join('@')) + end +end diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb index 59ea162c0e..ecc7f5d98e 100755 --- a/services/api/script/crunch-dispatch.rb +++ b/services/api/script/crunch-dispatch.rb @@ -144,16 +144,17 @@ class Dispatcher api_client_id: 0) job_auth.save - cmd_args << (ENV['CRUNCH_JOB_BIN'] || `which crunch-job`.strip) + crunch_job_bin = (ENV['CRUNCH_JOB_BIN'] || `which arv-crunch-job`.strip) + if crunch_job_bin == '' + raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path." + end + + cmd_args << crunch_job_bin cmd_args << '--job-api-token' cmd_args << job_auth.api_token cmd_args << '--job' cmd_args << job.uuid - if cmd_args[0] == '' - raise "No CRUNCH_JOB_BIN env var, and crunch-job not in path." - end - commit = Commit.where(sha1: job.script_version).first if commit cmd_args << '--git-dir' diff --git a/services/api/test/unit/blob_test.rb b/services/api/test/unit/blob_test.rb new file mode 100644 index 0000000000..ec6e67a168 --- /dev/null +++ b/services/api/test/unit/blob_test.rb @@ -0,0 +1,94 @@ +require 'test_helper' + +class BlobTest < ActiveSupport::TestCase + @@api_token = rand(2**512).to_s(36)[0..49] + @@key = rand(2**2048).to_s(36) + @@blob_data = 'foo' + @@blob_locator = Digest::MD5.hexdigest(@@blob_data) + + '+' + @@blob_data.size.to_s + + test 'correct' do + signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key + assert_equal true, Blob.verify_signature!(signed, api_token: @@api_token, key: @@key) + end + + test 'expired' do + signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key, ttl: -1 + assert_raise Blob::InvalidSignatureError do + Blob.verify_signature!(signed, api_token: @@api_token, key: @@key) + end + end + + test 'expired, but no raise' do + signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key, ttl: -1 + assert_equal false, Blob.verify_signature(signed, + api_token: @@api_token, + key: @@key) + end + + test 'bogus, wrong block hash' do + signed = Blob.sign_locator @@blob_locator, api_token: @@api_token, key: @@key + assert_raise Blob::InvalidSignatureError do + Blob.verify_signature!(signed.sub('acbd','abcd'), api_token: @@api_token, key: @@key) + end + end + + test 'bogus, expired' do + signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@531641bf' + assert_raises Blob::InvalidSignatureError do + Blob.verify_signature!(signed, api_token: @@api_token, key: @@key) + end + end + + test 'bogus, wrong key' do + signed = Blob.sign_locator(@@blob_locator, + api_token: @@api_token, + key: (@@key+'x')) + assert_raise Blob::InvalidSignatureError do + Blob.verify_signature!(signed, api_token: @@api_token, key: @@key) + end + end + + test 'bogus, wrong api token' do + signed = Blob.sign_locator(@@blob_locator, + api_token: @@api_token.reverse, + key: @@key) + assert_raise Blob::InvalidSignatureError do + Blob.verify_signature!(signed, api_token: @@api_token, key: @@key) + end + end + + test 'bogus, signature format 1' do + signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@' + assert_raise Blob::InvalidSignatureError do + Blob.verify_signature!(signed, api_token: @@api_token, key: @@key) + end + end + + test 'bogus, signature format 2' do + signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+A@531641bf' + assert_raise Blob::InvalidSignatureError do + Blob.verify_signature!(signed, api_token: @@api_token, key: @@key) + end + end + + test 'bogus, signature format 3' do + signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Axyzzy@531641bf' + assert_raise Blob::InvalidSignatureError do + Blob.verify_signature!(signed, api_token: @@api_token, key: @@key) + end + end + + test 'bogus, timestamp format' do + signed = 'acbd18db4cc2f85cedef654fccc4a4d8+3+Aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa@xyzzy' + assert_raise Blob::InvalidSignatureError do + Blob.verify_signature!(signed, api_token: @@api_token, key: @@key) + end + end + + test 'no signature at all' do + assert_raise Blob::InvalidSignatureError do + Blob.verify_signature!(@@blob_locator, api_token: @@api_token, key: @@key) + end + end +end