14259: Add method to keep client to request remote block copy via HEAD request.

Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <ldipentima@veritasgenetics.com>

diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 71e101cf4c5073d40e78f73c0bf46a9ff231f937..9618ee5ce9ac5551931093b24695514379e1376f 100644 (file)
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -421,6 +421,10 @@ class KeepClient(object):
                 _logger.info("HEAD %s: %s bytes",
                          self._result['status_code'],
                          self._result.get('content-length'))
+                if self._result['headers'].get('x-keep-locator'):
+                    # This is a response to a remote block copy request, return
+                    # the local copy block locator.
+                    return self._result['headers'].get('x-keep-locator')
                 return True
 
             _logger.info("GET %s: %s bytes in %s msec (%.3f MiB/sec)",
@@ -975,6 +979,11 @@ class KeepClient(object):
         else:
             return None
 
+    def refresh_signature(self, loc):
+        """Ask Keep to get the remote block and return its local signature"""
+        now = datetime.datetime.utcnow().isoformat("T") + 'Z'
+        return self.head(loc, headers={'X-Keep-Signature': 'local, {}'.format(now)})
+
     @retry.retry_method
     def head(self, loc_s, **kwargs):
         return self._get_or_head(loc_s, method="HEAD", **kwargs)
@@ -983,7 +992,7 @@ class KeepClient(object):
     def get(self, loc_s, **kwargs):
         return self._get_or_head(loc_s, method="GET", **kwargs)
 
-    def _get_or_head(self, loc_s, method="GET", num_retries=None, request_id=None):
+    def _get_or_head(self, loc_s, method="GET", num_retries=None, request_id=None, headers=None):
         """Get data from Keep.
 
         This method fetches one or more blocks of data from Keep.  It
@@ -1024,11 +1033,11 @@ class KeepClient(object):
 
             self.misses_counter.add(1)
 
-            headers = {
-                'X-Request-Id': (request_id or
-                                 (hasattr(self, 'api_client') and self.api_client.request_id) or
-                                 arvados.util.new_request_id()),
-            }
+            if headers is None:
+                headers = {}
+            headers['X-Request-Id'] = (request_id or
+                                        (hasattr(self, 'api_client') and self.api_client.request_id) or
+                                        arvados.util.new_request_id())
 
             # If the locator has hints specifying a prefix (indicating a
             # remote keepproxy) or the UUID of a local gateway service,
@@ -1086,7 +1095,7 @@ class KeepClient(object):
             # Always cache the result, then return it if we succeeded.
             if loop.success():
                 if method == "HEAD":
-                    return True
+                    return blob or True
                 else:
                     return blob
         finally:

diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index a7b79933bbc2999381fea887ac3a70e77f346b3c..a6211e7ea939a92bc0497cf2555ce105389dc503 100644 (file)
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -934,6 +934,16 @@ class KeepClientGatewayTestCase(unittest.TestCase, tutil.ApiClientMock):
                          MockCurl.return_value.getopt(pycurl.URL).decode())
         self.assertEqual(True, self.keepClient.head(locator))
 
+    @mock.patch('pycurl.Curl')
+    def test_refresh_signature(self, MockCurl):
+        blk_digest = '6f5902ac237024bdd0c176cb93063dc4+11'
+        blk_sig = 'da39a3ee5e6b4b0d3255bfef95601890afd80709@53bed294'
+        MockCurl.return_value = tutil.FakeCurl.make(
+            code=200, body='', headers={'X-Keep-Locator':blk_digest+'+A'+blk_sig})
+        self.mock_disks_and_gateways()
+        locator = blk_digest+'+R'+blk_sig
+        self.assertEqual(blk_digest+'+A'+blk_sig, self.keepClient.refresh_signature(locator))
+
     @mock.patch('pycurl.Curl')
     def test_get_with_gateway_hints_in_order(self, MockCurl):
         gateways = 4