import time
import threading
import timer
+import datetime
global_client_object = None
from api import *
import config
import arvados.errors
+import arvados.util
+
+class KeepLocator(object):
+ EPOCH_DATETIME = datetime.datetime.utcfromtimestamp(0)
+
+ def __init__(self, locator_str):
+ self.size = None
+ self.loc_hint = None
+ self._perm_sig = None
+ self._perm_expiry = None
+ pieces = iter(locator_str.split('+'))
+ self.md5sum = next(pieces)
+ for hint in pieces:
+ if hint.startswith('A'):
+ self.parse_permission_hint(hint)
+ elif hint.startswith('K'):
+ self.loc_hint = hint # FIXME
+ elif hint.isdigit():
+ self.size = int(hint)
+ else:
+ raise ValueError("unrecognized hint data {}".format(hint))
+
+ def __str__(self):
+ return '+'.join(
+ str(s) for s in [self.md5sum, self.size, self.loc_hint,
+ self.permission_hint()]
+ if s is not None)
+
+ def _make_hex_prop(name, length):
+ # Build and return a new property with the given name that
+ # must be a hex string of the given length.
+ data_name = '_{}'.format(name)
+ def getter(self):
+ return getattr(self, data_name)
+ def setter(self, hex_str):
+ if not arvados.util.is_hex(hex_str, length):
+ raise ValueError("{} must be a {}-digit hex string: {}".
+ format(name, length, hex_str))
+ setattr(self, data_name, hex_str)
+ return property(getter, setter)
+
+ md5sum = _make_hex_prop('md5sum', 32)
+ perm_sig = _make_hex_prop('perm_sig', 40)
+
+ @property
+ def perm_expiry(self):
+ return self._perm_expiry
+
+ @perm_expiry.setter
+ def perm_expiry(self, value):
+ if not arvados.util.is_hex(value, 1, 8):
+ raise ValueError(
+ "permission timestamp must be a hex Unix timestamp: {}".
+ format(value))
+ self._perm_expiry = datetime.datetime.utcfromtimestamp(int(value, 16))
+
+ def permission_hint(self):
+ data = [self.perm_sig, self.perm_expiry]
+ if None in data:
+ return None
+ data[1] = int((data[1] - self.EPOCH_DATETIME).total_seconds())
+ return "A{}@{:08x}".format(*data)
+
+ def parse_permission_hint(self, s):
+ try:
+ self.perm_sig, self.perm_expiry = s[1:].split('@', 1)
+ except IndexError:
+ raise ValueError("bad permission hint {}".format(s))
+
+ def permission_expired(self, as_of_dt=None):
+ if self.perm_expiry is None:
+ return False
+ elif as_of_dt is None:
+ as_of_dt = datetime.datetime.now()
+ return self.perm_expiry <= as_of_dt
+
class Keep:
@staticmethod
def __init__(self, todo):
self._todo = todo
self._done = 0
+ self._response = None
self._todo_lock = threading.Semaphore(todo)
self._done_lock = threading.Lock()
with self._done_lock:
return (self._done < self._todo)
- def increment_done(self):
+ def save_response(self, response_body, replicas_stored):
"""
- Report that the current thread was successful.
+ Records a response body (a locator, possibly signed) returned by
+ the Keep server. It is not necessary to save more than
+ one response, since we presume that any locator returned
+ in response to a successful request is valid.
"""
with self._done_lock:
- self._done += 1
+ self._done += replicas_stored
+ self._response = response_body
+
+ def response(self):
+ """
+ Returns the body from the response to a PUT request.
+ """
+ with self._done_lock:
+ return self._response
def done(self):
"""
class KeepWriterThread(threading.Thread):
"""
- Write a blob of data to the given Keep server. Call
- increment_done() of the given ThreadLimiter if the write
- succeeds.
+ Write a blob of data to the given Keep server. On success, call
+ save_response() of the given ThreadLimiter to save the returned
+ locator.
"""
def __init__(self, **kwargs):
super(KeepClient.KeepWriterThread, self).__init__()
headers['X-Keep-Desired-Replication'] = str(self.args['want_copies'])
try:
+ logging.debug("Uploading to {}".format(url))
resp, content = h.request(url.encode('utf-8'), 'PUT',
headers=headers,
body=self.args['data'])
(str(threading.current_thread()),
self.args['data_hash'],
self.args['service_root']))
-
+ replicas_stored = 1
if 'x-keep-replicas-stored' in resp:
# Tick the 'done' counter for the number of replica
# reported stored by the server, for the case that
# we're talking to a proxy or other backend that
# stores to multiple copies for us.
- replicas = int(resp['x-keep-replicas-stored'])
- while replicas > 0:
- limiter.increment_done()
- replicas -= 1
- else:
- limiter.increment_done()
- return
+ try:
+ replicas_stored = int(resp['x-keep-replicas-stored'])
+ except ValueError:
+ pass
+ return limiter.save_response(content.strip(), replicas_stored)
+
logging.warning("Request fail: PUT %s => %s %s" %
(url, resp['status'], content))
except (httplib2.HttpLib2Error, httplib.HTTPException) as e:
# Override normal keep disk lookup with an explict proxy
# configuration.
keep_proxy_env = config.get("ARVADOS_KEEP_PROXY")
- if keep_proxy_env != None:
+ if keep_proxy_env != None and len(keep_proxy_env) > 0:
+
if keep_proxy_env[-1:] != '/':
keep_proxy_env += "/"
self.service_roots = [keep_proxy_env]
try:
try:
keep_services = arvados.api().keep_services().accessible().execute()['items']
- except:
+ except Exception:
keep_services = arvados.api().keep_disks().list().execute()['items']
if len(keep_services) == 0:
finally:
self.lock.release()
+ # Build an ordering with which to query the Keep servers based on the
+ # contents of the hash.
+ # "hash" is a hex-encoded number at least 8 digits
+ # (32 bits) long
+
+ # seed used to calculate the next keep server from 'pool'
+ # to be added to 'pseq'
seed = hash
+
+ # Keep servers still to be added to the ordering
pool = self.service_roots[:]
+
+ # output probe sequence
pseq = []
+
+ # iterate while there are servers left to be assigned
while len(pool) > 0:
if len(seed) < 8:
- if len(pseq) < len(hash) / 4: # first time around
+ # ran out of digits in the seed
+ if len(pseq) < len(hash) / 4:
+ # the number of servers added to the probe sequence is less
+ # than the number of 4-digit slices in 'hash' so refill the
+ # seed with the last 4 digits and then append the contents
+ # of 'hash'.
seed = hash[-4:] + hash
else:
+ # refill the seed with the contents of 'hash'
seed += hash
+
+ # Take the next 8 digits (32 bytes) and interpret as an integer,
+ # then modulus with the size of the remaining pool to get the next
+ # selected server.
probe = int(seed[0:8], 16) % len(pool)
+
+ # Append the selected server to the probe sequence and remove it
+ # from the pool.
pseq += [pool[probe]]
pool = pool[:probe] + pool[probe+1:]
+
+ # Remove the digits just used from the seed
seed = seed[8:]
logging.debug(str(pseq))
return pseq
try:
for service_root in self.shuffled_service_roots(expect_hash):
- url = service_root + expect_hash
+ url = service_root + locator
api_token = config.get('ARVADOS_API_TOKEN')
headers = {'Authorization': "OAuth2 %s" % api_token,
'Accept': 'application/octet-stream'}
for location_hint in re.finditer(r'\+K@([a-z0-9]+)', locator):
instance = location_hint.group(1)
- url = 'http://keep.' + instance + '.arvadosapi.com/' + expect_hash
+ url = 'http://keep.' + instance + '.arvadosapi.com/' + locator
blob = self.get_url(url, {}, expect_hash)
if blob:
slot.set(blob)
for t in threads:
t.join()
have_copies = thread_limiter.done()
+ # If we're done, return the response from Keep
if have_copies >= want_copies:
- return (data_hash + '+' + str(len(data)))
+ return thread_limiter.response()
raise arvados.errors.KeepWriteError(
"Write fail for %s: wanted %d but wrote %d" %
(data_hash, want_copies, have_copies))