X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0e0c1400b57d5de8aa8c18dd4897527f905a4b42..c02ceff00fce94ec5794b53fe890f681acf31121:/sdk/python/arvados/keep.py diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py index ee91491efa..351f7f5dda 100644 --- a/sdk/python/arvados/keep.py +++ b/sdk/python/arvados/keep.py @@ -1,14 +1,20 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + from __future__ import absolute_import from __future__ import division from future import standard_library +from future.utils import native_str standard_library.install_aliases() from builtins import next from builtins import str from builtins import range from builtins import object -import io +import collections import datetime import hashlib +import io import logging import math import os @@ -72,8 +78,9 @@ class KeepLocator(object): def __str__(self): return '+'.join( - str(s) for s in [self.md5sum, self.size, - self.permission_hint()] + self.hints + native_str(s) + for s in [self.md5sum, self.size, + self.permission_hint()] + self.hints if s is not None) def stripped(self): @@ -90,7 +97,7 @@ class KeepLocator(object): return getattr(self, data_name) def setter(self, hex_str): if not arvados.util.is_hex(hex_str, length): - raise ValueError("{} is not a {}-digit hex string: {}". + raise ValueError("{} is not a {}-digit hex string: {!r}". format(name, length, hex_str)) setattr(self, data_name, hex_str) return property(getter, setter) @@ -284,12 +291,14 @@ class KeepClient(object): def __init__(self, root, user_agent_pool=queue.LifoQueue(), upload_counter=None, - download_counter=None, **headers): + download_counter=None, + headers={}): self.root = root self._user_agent_pool = user_agent_pool self._result = {'error': None} self._usable = True self._session = None + self._socket = None self.get_headers = {'Accept': 'application/octet-stream'} self.get_headers.update(headers) self.put_headers = headers @@ -320,15 +329,28 @@ class KeepClient(object): except: ua.close() - @staticmethod - def _socket_open(family, socktype, protocol, address=None): + def _socket_open(self, *args, **kwargs): + if len(args) + len(kwargs) == 2: + return self._socket_open_pycurl_7_21_5(*args, **kwargs) + else: + return self._socket_open_pycurl_7_19_3(*args, **kwargs) + + def _socket_open_pycurl_7_19_3(self, family, socktype, protocol, address=None): + return self._socket_open_pycurl_7_21_5( + purpose=None, + address=collections.namedtuple( + 'Address', ['family', 'socktype', 'protocol', 'addr'], + )(family, socktype, protocol, address)) + + def _socket_open_pycurl_7_21_5(self, purpose, address): """Because pycurl doesn't have CURLOPT_TCP_KEEPALIVE""" - s = socket.socket(family, socktype, protocol) + s = socket.socket(address.family, address.socktype, address.protocol) s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) # Will throw invalid protocol error on mac. This test prevents that. if hasattr(socket, 'TCP_KEEPIDLE'): s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 75) s.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, 75) + self._socket = s return s def get(self, locator, method="GET", timeout=None): @@ -342,7 +364,8 @@ class KeepClient(object): self._headers = {} response_body = BytesIO() curl.setopt(pycurl.NOSIGNAL, 1) - curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open) + curl.setopt(pycurl.OPENSOCKETFUNCTION, + lambda *args, **kwargs: self._socket_open(*args, **kwargs)) curl.setopt(pycurl.URL, url.encode('utf-8')) curl.setopt(pycurl.HTTPHEADER, [ '{}: {}'.format(k,v) for k,v in self.get_headers.items()]) @@ -356,6 +379,10 @@ class KeepClient(object): curl.perform() except Exception as e: raise arvados.errors.HttpError(0, str(e)) + finally: + if self._socket: + self._socket.close() + self._socket = None self._result = { 'status_code': curl.getinfo(pycurl.RESPONSE_CODE), 'body': response_body.getvalue(), @@ -420,7 +447,8 @@ class KeepClient(object): body_reader = BytesIO(body) response_body = BytesIO() curl.setopt(pycurl.NOSIGNAL, 1) - curl.setopt(pycurl.OPENSOCKETFUNCTION, self._socket_open) + curl.setopt(pycurl.OPENSOCKETFUNCTION, + lambda *args, **kwargs: self._socket_open(*args, **kwargs)) curl.setopt(pycurl.URL, url.encode('utf-8')) # Using UPLOAD tells cURL to wait for a "go ahead" from the # Keep server (in the form of a HTTP/1.1 "100 Continue" @@ -440,9 +468,13 @@ class KeepClient(object): curl.perform() except Exception as e: raise arvados.errors.HttpError(0, str(e)) + finally: + if self._socket: + self._socket.close() + self._socket = None self._result = { 'status_code': curl.getinfo(pycurl.RESPONSE_CODE), - 'body': response_body.getvalue(), + 'body': response_body.getvalue().decode('utf-8'), 'headers': self._headers, 'error': False, } @@ -851,7 +883,7 @@ class KeepClient(object): The weight is md5(h + u) where u is the last 15 characters of the service endpoint's UUID. """ - return hashlib.md5(data_hash + service_uuid[-15:]).hexdigest() + return hashlib.md5((data_hash + service_uuid[-15:]).encode()).hexdigest() def weighted_service_roots(self, locator, force_rebuild=False, need_writable=False): """Return an array of Keep service endpoints, in the order in @@ -889,7 +921,7 @@ class KeepClient(object): _logger.debug("{}: {}".format(locator, sorted_roots)) return sorted_roots - def map_new_services(self, roots_map, locator, force_rebuild, need_writable, **headers): + def map_new_services(self, roots_map, locator, force_rebuild, need_writable, headers): # roots_map is a dictionary, mapping Keep service root strings # to KeepService objects. Poll for Keep services, and add any # new ones to roots_map. Return the current list of local @@ -902,7 +934,7 @@ class KeepClient(object): root, self._user_agent_pool, upload_counter=self.upload_counter, download_counter=self.download_counter, - **headers) + headers=headers) return local_roots @staticmethod @@ -932,14 +964,14 @@ class KeepClient(object): return None @retry.retry_method - def head(self, loc_s, num_retries=None): - return self._get_or_head(loc_s, method="HEAD", num_retries=num_retries) + def head(self, loc_s, **kwargs): + return self._get_or_head(loc_s, method="HEAD", **kwargs) @retry.retry_method - def get(self, loc_s, num_retries=None): - return self._get_or_head(loc_s, method="GET", num_retries=num_retries) + def get(self, loc_s, **kwargs): + return self._get_or_head(loc_s, method="GET", **kwargs) - def _get_or_head(self, loc_s, method="GET", num_retries=None): + def _get_or_head(self, loc_s, method="GET", num_retries=None, request_id=None): """Get data from Keep. This method fetches one or more blocks of data from Keep. It @@ -974,6 +1006,12 @@ class KeepClient(object): self.misses_counter.add(1) + headers = { + 'X-Request-Id': (request_id or + (hasattr(self, 'api_client') and self.api_client.request_id) or + arvados.util.new_request_id()), + } + # If the locator has hints specifying a prefix (indicating a # remote keepproxy) or the UUID of a local gateway service, # read data from the indicated service(s) instead of the usual @@ -990,7 +1028,8 @@ class KeepClient(object): roots_map = { root: self.KeepService(root, self._user_agent_pool, upload_counter=self.upload_counter, - download_counter=self.download_counter) + download_counter=self.download_counter, + headers=headers) for root in hint_roots } @@ -1009,7 +1048,8 @@ class KeepClient(object): sorted_roots = self.map_new_services( roots_map, locator, force_rebuild=(tries_left < num_retries), - need_writable=False) + need_writable=False, + headers=headers) except Exception as error: loop.save_result(error) continue @@ -1053,7 +1093,7 @@ class KeepClient(object): "failed to read {}".format(loc_s), service_errors, label="service") @retry.retry_method - def put(self, data, copies=2, num_retries=None): + def put(self, data, copies=2, num_retries=None, request_id=None): """Save data in Keep. This method will get a list of Keep services from the API server, and @@ -1083,9 +1123,12 @@ class KeepClient(object): return loc_s locator = KeepLocator(loc_s) - headers = {} - # Tell the proxy how many copies we want it to store - headers['X-Keep-Desired-Replicas'] = str(copies) + headers = { + 'X-Request-Id': (request_id or + (hasattr(self, 'api_client') and self.api_client.request_id) or + arvados.util.new_request_id()), + 'X-Keep-Desired-Replicas': str(copies), + } roots_map = {} loop = retry.RetryLoop(num_retries, self._check_loop_result, backoff_start=2) @@ -1094,7 +1137,9 @@ class KeepClient(object): try: sorted_roots = self.map_new_services( roots_map, locator, - force_rebuild=(tries_left < num_retries), need_writable=True, **headers) + force_rebuild=(tries_left < num_retries), + need_writable=True, + headers=headers) except Exception as error: loop.save_result(error) continue @@ -1141,7 +1186,7 @@ class KeepClient(object): """ md5 = hashlib.md5(data).hexdigest() locator = '%s+%d' % (md5, len(data)) - with open(os.path.join(self.local_store, md5 + '.tmp'), 'w') as f: + with open(os.path.join(self.local_store, md5 + '.tmp'), 'wb') as f: f.write(data) os.rename(os.path.join(self.local_store, md5 + '.tmp'), os.path.join(self.local_store, md5)) @@ -1155,8 +1200,8 @@ class KeepClient(object): raise arvados.errors.NotFoundError( "Invalid data locator: '%s'" % loc_s) if locator.md5sum == config.EMPTY_BLOCK_LOCATOR.split('+')[0]: - return '' - with open(os.path.join(self.local_store, locator.md5sum), 'r') as f: + return b'' + with open(os.path.join(self.local_store, locator.md5sum), 'rb') as f: return f.read() def is_cached(self, locator):