Merge branch 'master' into 4823-python-sdk-writable-collection-api

Conflicts:
	sdk/python/arvados/collection.py
	sdk/python/tests/test_keep_client.py

diff --cc sdk/python/arvados/collection.py
index 20f5c40bfccdf8eb945db95bf575e6f2396782f7,7bfdf782f8d06b03d6ac482fa64872d1eb8ff9be..82708cc02ebd5b99cd96b3a3095c934e4d95b2ea
--- 1/sdk/python/arvados/collection.py
--- 2/sdk/python/arvados/collection.py
+++ b/sdk/python/arvados/collection.py
@@@ -268,7 -302,9 +268,7 @@@ class _WriterFile(ArvadosFileBase)
  
  
  class CollectionWriter(CollectionBase):
-     def __init__(self, api_client=None, num_retries=0):
 -    KEEP_BLOCK_SIZE = 2**26
 -
+     def __init__(self, api_client=None, num_retries=0, replication=None):
          """Instantiate a CollectionWriter.
  
          CollectionWriter lets you build a new Arvados Collection from scratch.
@@@ -441,8 -481,10 +445,10 @@@
          data_buffer = ''.join(self._data_buffer)
          if data_buffer:
              self._current_stream_locators.append(
-                 self._my_keep().put(data_buffer[0:config.KEEP_BLOCK_SIZE]))
+                 self._my_keep().put(
 -                    data_buffer[0:self.KEEP_BLOCK_SIZE],
++                    data_buffer[0:config.KEEP_BLOCK_SIZE],
+                     copies=self.replication))
 -            self._data_buffer = [data_buffer[self.KEEP_BLOCK_SIZE:]]
 +            self._data_buffer = [data_buffer[config.KEEP_BLOCK_SIZE:]]
              self._data_buffer_len = len(self._data_buffer[0])
  
      def start_new_file(self, newfilename=None):

diff --cc sdk/python/arvados/commands/put.py
Simple merge

diff --cc sdk/python/arvados/keep.py
Simple merge

diff --cc sdk/python/tests/arvados_testutil.py
index 474c068b85d4eb05af274f381f0eb6be5a0789dd,124fb3570af36f21f4df974e162019bea599f569..aa0da982be2ed831982a6dccd22899f185eddb11
--- 1/sdk/python/tests/arvados_testutil.py
--- 2/sdk/python/tests/arvados_testutil.py
+++ b/sdk/python/tests/arvados_testutil.py
@@@ -76,11 -64,43 +77,45 @@@ class MockStreamReader(object)
          return self._name
  
      def readfrom(self, start, size, num_retries=None):
 -        return self._data[start:start + size]
 +        self._readfrom(start, size, num_retries=num_retries)
  
 +    def _readfrom(self, start, size, num_retries=None):
 +        return self._data[start:start + size]
  
+ class ApiClientMock(object):
+     def api_client_mock(self):
+         return mock.MagicMock(name='api_client_mock')
+ 
+     def mock_keep_services(self, api_mock=None, status=200, count=12,
+                            service_type='disk',
+                            service_host=None,
+                            service_port=None,
+                            service_ssl_flag=False):
+         if api_mock is None:
+             api_mock = self.api_client_mock()
+         body = {
+             'items_available': count,
+             'items': [{
+                 'uuid': 'zzzzz-bi6l4-{:015x}'.format(i),
+                 'owner_uuid': 'zzzzz-tpzed-000000000000000',
+                 'service_host': service_host or 'keep0x{:x}'.format(i),
+                 'service_port': service_port or 65535-i,
+                 'service_ssl_flag': service_ssl_flag,
+                 'service_type': service_type,
+             } for i in range(0, count)]
+         }
+         self._mock_api_call(api_mock.keep_services().accessible, status, body)
+         return api_mock
+ 
+     def _mock_api_call(self, mock_method, code, body):
+         mock_method = mock_method().execute
+         if code == 200:
+             mock_method.return_value = body
+         else:
+             mock_method.side_effect = arvados.errors.ApiError(
+                 fake_httplib2_response(code), "{}")
+ 
+ 
  class ArvadosBaseTestCase(unittest.TestCase):
      # This class provides common utility functions for our tests.
  

diff --cc sdk/python/tests/test_collections.py
Simple merge

diff --cc sdk/python/tests/test_keep_client.py
index 885ae7c1d0bd66acf25be72a1fd623607db9892d,6d4d3cd8b816ea3854151285cd46cdebf60f5e63..1baf1357ccf8dbfdc78960d4a63b326595e249c5
--- 1/sdk/python/tests/test_keep_client.py
--- 2/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@@ -277,58 -257,58 +257,58 @@@ class KeepClientServiceTestCase(unittes
      # when connected directly to a Keep server (i.e. non-proxy timeout)
  
      def test_get_timeout(self):
-         api_client = self.mock_keep_services(('keep', 10, False, 'disk'))
+         api_client = self.mock_keep_services(count=1)
 -        keep_client = arvados.KeepClient(api_client=api_client)
          force_timeout = [socket.timeout("timed out")]
 -        with mock.patch('requests.get', side_effect=force_timeout) as mock_request:
 +        with tutil.mock_get(force_timeout) as mock_session:
 +            keep_client = arvados.KeepClient(api_client=api_client)
              with self.assertRaises(arvados.errors.KeepReadError):
                  keep_client.get('ffffffffffffffffffffffffffffffff')
 -            self.assertTrue(mock_request.called)
 +            self.assertTrue(mock_session.return_value.get.called)
              self.assertEqual(
                  arvados.KeepClient.DEFAULT_TIMEOUT,
 -                mock_request.call_args[1]['timeout'])
 +                mock_session.return_value.get.call_args[1]['timeout'])
  
      def test_put_timeout(self):
-         api_client = self.mock_keep_services(('keep', 10, False, 'disk'))
+         api_client = self.mock_keep_services(count=1)
 -        keep_client = arvados.KeepClient(api_client=api_client)
          force_timeout = [socket.timeout("timed out")]
 -        with mock.patch('requests.put', side_effect=force_timeout) as mock_request:
 +        with tutil.mock_put(force_timeout) as mock_session:
 +            keep_client = arvados.KeepClient(api_client=api_client)
              with self.assertRaises(arvados.errors.KeepWriteError):
                  keep_client.put('foo')
 -            self.assertTrue(mock_request.called)
 +            self.assertTrue(mock_session.return_value.put.called)
              self.assertEqual(
                  arvados.KeepClient.DEFAULT_TIMEOUT,
 -                mock_request.call_args[1]['timeout'])
 +                mock_session.return_value.put.call_args[1]['timeout'])
  
      def test_proxy_get_timeout(self):
          # Force a timeout, verifying that the requests.get or
          # requests.put method was called with the proxy_timeout
          # setting rather than the default timeout.
-         api_client = self.mock_keep_services(('keep', 10, False, 'proxy'))
+         api_client = self.mock_keep_services(service_type='proxy', count=1)
 -        keep_client = arvados.KeepClient(api_client=api_client)
          force_timeout = [socket.timeout("timed out")]
 -        with mock.patch('requests.get', side_effect=force_timeout) as mock_request:
 +        with tutil.mock_get(force_timeout) as mock_session:
 +            keep_client = arvados.KeepClient(api_client=api_client)
              with self.assertRaises(arvados.errors.KeepReadError):
                  keep_client.get('ffffffffffffffffffffffffffffffff')
 -            self.assertTrue(mock_request.called)
 +            self.assertTrue(mock_session.return_value.get.called)
              self.assertEqual(
                  arvados.KeepClient.DEFAULT_PROXY_TIMEOUT,
 -                mock_request.call_args[1]['timeout'])
 +                mock_session.return_value.get.call_args[1]['timeout'])
  
      def test_proxy_put_timeout(self):
          # Force a timeout, verifying that the requests.get or
          # requests.put method was called with the proxy_timeout
          # setting rather than the default timeout.
-         api_client = self.mock_keep_services(('keep', 10, False, 'proxy'))
+         api_client = self.mock_keep_services(service_type='proxy', count=1)
 -        keep_client = arvados.KeepClient(api_client=api_client)
          force_timeout = [socket.timeout("timed out")]
 -        with mock.patch('requests.put', side_effect=force_timeout) as mock_request:
 +        with tutil.mock_put(force_timeout) as mock_session:
 +            keep_client = arvados.KeepClient(api_client=api_client)
              with self.assertRaises(arvados.errors.KeepWriteError):
                  keep_client.put('foo')
 -            self.assertTrue(mock_request.called)
 +            self.assertTrue(mock_session.return_value.put.called)
              self.assertEqual(
                  arvados.KeepClient.DEFAULT_PROXY_TIMEOUT,
 -                mock_request.call_args[1]['timeout'])
 +                mock_session.return_value.put.call_args[1]['timeout'])
  
      def test_probe_order_reference_set(self):
          # expected_order[i] is the probe order for
@@@ -431,7 -413,8 +413,7 @@@
          self.check_no_services_error('put', arvados.errors.KeepWriteError)
  
      def check_errors_from_last_retry(self, verb, exc_class):
-         api_client = self.mock_n_keep_disks(2)
+         api_client = self.mock_keep_services(count=2)
 -        keep_client = arvados.KeepClient(api_client=api_client)
          req_mock = getattr(tutil, 'mock_{}_responses'.format(verb))(
              "retry error reporting test", 500, 500, 403, 403)
          with req_mock, tutil.skip_sleep, \
@@@ -452,10 -434,10 +434,10 @@@
      def test_put_error_does_not_include_successful_puts(self):
          data = 'partial failure test'
          data_loc = '{}+{}'.format(hashlib.md5(data).hexdigest(), len(data))
-         api_client = self.mock_n_keep_disks(3)
+         api_client = self.mock_keep_services(count=3)
 -        keep_client = arvados.KeepClient(api_client=api_client)
          with tutil.mock_put_responses(data_loc, 200, 500, 500) as req_mock, \
                  self.assertRaises(arvados.errors.KeepWriteError) as exc_check:
 +            keep_client = arvados.KeepClient(api_client=api_client)
              keep_client.put(data)
          self.assertEqual(2, len(exc_check.exception.service_errors()))
  

diff --cc services/fuse/arvados_fuse/__init__.py
Simple merge