roots = (("http%s://%s:%d/" %
('s' if f['service_ssl_flag'] else '',
f['service_host'],
roots = (("http%s://%s:%d/" %
('s' if f['service_ssl_flag'] else '',
f['service_host'],
if re.search(r',', locator):
return ''.join(self.get(x) for x in locator.split(','))
if re.search(r',', locator):
return ''.join(self.get(x) for x in locator.split(','))
expect_hash = re.sub(r'\+.*', '', locator)
slot, first = self.reserve_cache(expect_hash)
expect_hash = re.sub(r'\+.*', '', locator)
slot, first = self.reserve_cache(expect_hash)
- for service_root in self.shuffled_service_roots(expect_hash):
- url = service_root + expect_hash
- api_token = config.get('ARVADOS_API_TOKEN')
- headers = {'Authorization': "OAuth2 %s" % api_token,
- 'Accept': 'application/octet-stream'}
- blob = self.get_url(url, headers, expect_hash)
- if blob:
- slot.set(blob)
- self.cap_cache()
- return blob
-
- for location_hint in re.finditer(r'\+K@([a-z0-9]+)', locator):
- instance = location_hint.group(1)
- url = 'http://keep.' + instance + '.arvadosapi.com/' + expect_hash
- blob = self.get_url(url, {}, expect_hash)
- if blob:
- slot.set(blob)
- self.cap_cache()
- return blob
+ try:
+ for service_root in self.shuffled_service_roots(expect_hash):
+ url = service_root + expect_hash
+ api_token = config.get('ARVADOS_API_TOKEN')
+ headers = {'Authorization': "OAuth2 %s" % api_token,
+ 'Accept': 'application/octet-stream'}
+ blob = self.get_url(url, headers, expect_hash)
+ if blob:
+ slot.set(blob)
+ self.cap_cache()
+ return blob
+
+ for location_hint in re.finditer(r'\+K@([a-z0-9]+)', locator):
+ instance = location_hint.group(1)
+ url = 'http://keep.' + instance + '.arvadosapi.com/' + expect_hash
+ blob = self.get_url(url, {}, expect_hash)
+ if blob:
+ slot.set(blob)
+ self.cap_cache()
+ return blob
+ except:
+ slot.set(None)
+ self.cap_cache()
+ raise
raise arvados.errors.NotFoundError("Block not found: %s" % expect_hash)
def get_url(self, url, headers, expect_hash):
raise arvados.errors.NotFoundError("Block not found: %s" % expect_hash)
def get_url(self, url, headers, expect_hash):
with timer.Timer() as t:
resp, content = h.request(url.encode('utf-8'), 'GET',
headers=headers)
with timer.Timer() as t:
resp, content = h.request(url.encode('utf-8'), 'GET',
headers=headers)
- logging.info("Received %s bytes in %s msec (%s bytes/sec)" % (len(content), t.msecs, len(content)/t.secs))
+ logging.info("Received %s bytes in %s msec (%s MiB/sec)" % (len(content),
+ t.msecs,
+ (len(content)/(1024*1024))/t.secs))