2755: add support to arv-put for signed manifests.

When arv-put finishes a stream, the manifest it stores in Keep now has
been stripped of signatures and other variable hints.

test_cmdline.py tests arv-put's handling of the manifest to make sure
that, when permissions are enabled, the manifest in Keep lacks
signatures, and the same manifest returned from the API server includes
signatures.

Refs #2755.

diff --git a/sdk/python/arvados/collection.py b/sdk/python/arvados/collection.py
index fb3dea43acc6872d11ccdbc5456c2d87e5c15c73..ab0b0288bba33453941acccb8907c7449649251b 100644 (file)
--- a/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@ -156,6 +156,7 @@ class CollectionWriter(object):
         self._current_stream_name = '.'
         self._current_file_name = None
         self._current_file_pos = 0
+        self._manifest_text = ''
         self._finished_streams = []
 
     def __enter__(self):
@@ -272,9 +273,31 @@ class CollectionWriter(object):
         self._current_file_name = None
 
     def finish(self):
-        return Keep.put(self.manifest_text())
-
+        # Send the stripped manifest to Keep, to ensure that we use the
+        # same UUID regardless of what hints are used on the collection.
+        return Keep.put(self.stripped_manifest())
+
+    def stripped_manifest(self):
+        """
+        Return the manifest for the current collection with all hints
+        (other than size) removed from the locators in the manifest.
+        """
+        raw = self.manifest_text()
+        clean = ''
+        for line in raw.split("\n"):
+            fields = line.split()
+            if len(fields) > 0:
+                locators = [ re.sub(r'\+[A-Z][a-z0-9@_-]+', '', x)
+                             for x in fields[1:-1] ]
+                clean += fields[0] + ' ' + ' '.join(locators) + ' ' + fields[-1] + "\n"
+        return clean
+        
     def manifest_text(self):
+        if self._manifest_text == '':
+            self._manifest_text = self.generate_manifest()
+        return self._manifest_text
+
+    def generate_manifest(self):
         self.finish_current_stream()
         manifest = ''
 

diff --git a/sdk/python/test_cmdline.py b/sdk/python/test_cmdline.py
new file mode 100644 (file)
index 0000000..a7029e2
--- /dev/null
+++ b/sdk/python/test_cmdline.py
@@ -0,0 +1,65 @@
+import os
+import re
+import subprocess
+import unittest
+import tempfile
+import yaml
+
+import arvados
+import run_test_server
+
+class ArvPutTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        try:
+            del os.environ['KEEP_LOCAL_STORE']
+        except KeyError:
+            pass
+
+        # Use the blob_signing_key from the Rails "test" configuration
+        # to provision the Keep server.
+        with open(os.path.join(os.path.dirname(__file__),
+                               run_test_server.ARV_API_SERVER_DIR,
+                               "config",
+                               "application.yml")) as f:
+            rails_config = yaml.load(f.read())
+        config_blob_signing_key = rails_config["test"]["blob_signing_key"]
+        run_test_server.run()
+        run_test_server.run_keep(blob_signing_key=config_blob_signing_key,
+                                 enforce_permissions=True)
+
+    @classmethod
+    def tearDownClass(cls):
+        run_test_server.stop()
+        run_test_server.stop_keep()
+
+    def test_ArvPutSignedManifest(self):
+        run_test_server.authorize_with('active')
+        for v in ["ARVADOS_API_HOST",
+                  "ARVADOS_API_HOST_INSECURE",
+                  "ARVADOS_API_TOKEN"]:
+            os.environ[v] = arvados.config.settings()[v]
+
+        datadir = tempfile.mkdtemp()
+        with open(os.path.join(datadir, "foo"), "w") as f:
+            f.write("The quick brown fox jumped over the lazy dog")
+        p = subprocess.Popen(["arv-put", datadir],
+                             stdout=subprocess.PIPE)
+        (arvout, arverr) = p.communicate()
+        self.assertEqual(arverr, None)
+
+        # The manifest UUID returned by arv-put must be signed.
+        manifest_uuid = arvout.strip()
+        self.assertRegexpMatches(manifest_uuid, r'\+A[0-9a-f]+@[0-9a-f]{8}')
+
+        # The manifest text stored in Keep must contain unsigned locators.
+        m = arvados.Keep.get(manifest_uuid)
+        self.assertEqual(m, ". 08a008a01d498c404b0c30852b39d3b8+44 0:44:foo\n")
+
+        # The manifest text stored in the API server under the same
+        # manifest UUID must use signed locators.
+        api = arvados.api('v1', cache=False)
+        c = api.collections().get(uuid=manifest_uuid).execute()
+        self.assertRegexpMatches(
+            c['manifest_text'],
+            r'^\. 08a008a01d498c404b0c30852b39d3b8\+44\+A[0-9a-f]+@[0-9a-f]+ 0:44:foo\n')

diff --git a/services/api/config/application.yml.example b/services/api/config/application.yml.example
index 030e23894f00a5436354d36cf946272bdea616a2..d6db3a4f8b7bcca3aea4c9d92974d9d131b307c5 100644 (file)
--- a/services/api/config/application.yml.example
+++ b/services/api/config/application.yml.example
@@ -38,11 +38,11 @@ production:
   # The version below is suitable for AWS.
   # Uncomment and change <%# to <%= to use it.
   # compute_node_nameservers: <%#
-    require 'net/http'
-    ['local', 'public'].collect do |iface|
-      Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
-    end << '172.16.0.23'
-  %>
+  #   require 'net/http'
+  #   ['local', 'public'].collect do |iface|
+  #     Net::HTTP.get(URI("http://169.254.169.254/latest/meta-data/#{iface}-ipv4")).match(/^[\d\.]+$/)[0]
+  #   end << '172.16.0.23'
+  # %>
 
 test:
   uuid_prefix: zzzzz