Merge branch 'master' into 3198-writable-fuse

Conflicts:
	sdk/python/arvados/arvfile.py
	sdk/python/arvados/keep.py
	services/fuse/arvados_fuse/__init__.py

diff --cc sdk/python/arvados/arvfile.py
index c0ef5810728e42776d4b57c4f4e6da0b03dd4618,2e0eaa8ce0f645ca305826851c5eb7ddccb434e2..c086e5fd3a5b51550f46dce842e3814bb987457e
--- 1/sdk/python/arvados/arvfile.py
--- 2/sdk/python/arvados/arvfile.py
+++ b/sdk/python/arvados/arvfile.py
@@@ -734,10 -713,11 +734,10 @@@ class ArvadosFile(object)
              self._segments = new_segs
              self._modified = True
          elif size > self.size():
 -            raise IOError("truncate() does not support extending the file size")
 -
 +            raise IOError(errno.EINVAL, "truncate() does not support extending the file size")
  
      def readfrom(self, offset, size, num_retries, exact=False):
-         """Read upto `size` bytes from the file starting at `offset`.
+         """Read up to `size` bytes from the file starting at `offset`.
  
          :exact:
           If False (default), return less data than requested if the read
@@@ -909,24 -873,11 +909,24 @@@ class ArvadosFileReader(ArvadosFileRead
  
      @_FileLikeObjectBase._before_close
      @retry_method
 -    def read(self, size, num_retries=None):
 -        """Read up to `size` bytes from the stream, starting at the current file position."""
 -        data = self.arvadosfile.readfrom(self._filepos, size, num_retries, exact=True)
 -        self._filepos += len(data)
 -        return data
 +    def read(self, size=None, num_retries=None):
 +        """Read up to `size` bytes from the file and return the result.
 +
 +        Starts at the current file position.  If `size` is None, read the
 +        entire remainder of the file.
 +        """
 +        if size is None:
 +            data = []
 +            rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
 +            while rd:
 +                data.append(rd)
 +                self._filepos += len(rd)
 +                rd = self.arvadosfile.readfrom(self._filepos, config.KEEP_BLOCK_SIZE, num_retries)
 +            return ''.join(data)
 +        else:
-             data = self.arvadosfile.readfrom(self._filepos, size, num_retries)
++            data = self.arvadosfile.readfrom(self._filepos, size, num_retries, exact=True)
 +            self._filepos += len(data)
 +            return data
  
      @_FileLikeObjectBase._before_close
      @retry_method

diff --cc sdk/python/tests/test_arvfile.py
index f89ac72e66779f14affe59c00b7cd8b314b3ae83,1701aa43caae5baaa3f25522230c5b9c4ac46994..c5dbb1692aee231a8336f2f3b423e6d1e25bc2f5
--- 1/sdk/python/tests/test_arvfile.py
--- 2/sdk/python/tests/test_arvfile.py
+++ b/sdk/python/tests/test_arvfile.py
@@@ -437,23 -437,28 +437,28 @@@ class ArvadosFileReaderTestCase(StreamF
              blocks[loc] = d
              stream.append(Range(loc, n, len(d)))
              n += len(d)
 -        af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 -        return ArvadosFileReader(af, "count.txt")
 +        af = ArvadosFile(ArvadosFileReaderTestCase.MockParent(blocks, nocache), "count.txt", stream=stream, segments=[Range(1, 0, 3), Range(6, 3, 3), Range(11, 6, 3)])
 +        return ArvadosFileReader(af)
  
-     def test_read_returns_first_block(self):
-         # read() calls will be aligned on block boundaries - see #3663.
+     def test_read_block_crossing_behavior(self):
+         # read() needs to return all the data requested if possible, even if it
+         # crosses uncached blocks: https://arvados.org/issues/5856
          sfile = self.make_count_reader(nocache=True)
-         self.assertEqual('123', sfile.read(10))
+         self.assertEqual('12345678', sfile.read(8))
  
      def test_successive_reads(self):
+         # Override StreamFileReaderTestCase.test_successive_reads
          sfile = self.make_count_reader(nocache=True)
-         for expect in ['123', '456', '789', '']:
-             self.assertEqual(expect, sfile.read(10))
+         self.assertEqual('1234', sfile.read(4))
+         self.assertEqual('5678', sfile.read(4))
+         self.assertEqual('9', sfile.read(4))
+         self.assertEqual('', sfile.read(4))
  
      def test_tell_after_block_read(self):
+         # Override StreamFileReaderTestCase.test_tell_after_block_read
          sfile = self.make_count_reader(nocache=True)
-         sfile.read(5)
-         self.assertEqual(3, sfile.tell())
+         self.assertEqual('12345678', sfile.read(8))
+         self.assertEqual(8, sfile.tell())
  
      def test_prefetch(self):
          keep = ArvadosFileWriterTestCase.MockKeep({"2e9ec317e197819358fbc43afca7d837+8": "01234567", "e8dc4081b13434b45189a720b77b6818+8": "abcdefgh"})

diff --cc sdk/python/tests/test_collections.py
Simple merge

diff --cc services/fuse/arvados_fuse/__init__.py
index 1b44eafabb86d0d5775dc576b451fe003682600d,8d048487928c54f57dd944c083ee76ffb05c01d1..b85e65d74f7c3575c40a58c3901718fe9bd88419
--- 1/services/fuse/arvados_fuse/__init__.py
--- 2/services/fuse/arvados_fuse/__init__.py
+++ b/services/fuse/arvados_fuse/__init__.py
@@@ -198,12 -156,11 +198,11 @@@ class Operations(llfuse.Operations)
  
      """
  
-     def __init__(self, uid, gid,
-                  encoding="utf-8",
-                  inode_cache=InodeCache(cap=256*1024*1024),
-                  num_retries=4):
 -    def __init__(self, uid, gid, encoding="utf-8", inode_cache=None):
++    def __init__(self, uid, gid, encoding="utf-8", inode_cache=None, num_retries=4):
          super(Operations, self).__init__()
  
+         if not inode_cache:
+             inode_cache = InodeCache(cap=256*1024*1024)
          self.inodes = Inodes(inode_cache)
          self.uid = uid
          self.gid = gid

diff --cc services/fuse/arvados_fuse/fusedir.py
Simple merge

diff --cc services/fuse/tests/test_inodes.py
index 2c5f3dcef89efec4c265753f5895b99a14daca5e,61a365f8b06c9c7d5294d4662aa09aa118458191..cce5b95a80ed5c5ab0dfc6819ff96bb661f6387d
--- 1/services/fuse/tests/test_inodes.py
--- 2/services/fuse/tests/test_inodes.py
+++ b/services/fuse/tests/test_inodes.py
@@@ -79,9 -120,26 +120,27 @@@ class InodeTests(unittest.TestCase)
          self.assertTrue(ent3.clear.called)
          self.assertEqual(500, cache.total())
  
+     def test_delete(self):
+         cache = arvados_fuse.InodeCache(1000, 4)
+         inodes = arvados_fuse.Inodes(cache)
+ 
+         ent1 = mock.MagicMock()
+         ent1.in_use.return_value = False
+         ent1.persisted.return_value = True
+         ent1.clear.return_value = True
+         ent1.objsize.return_value = 500
+         inodes.add_entry(ent1)
+ 
+         ent3 = mock.MagicMock()
+         ent3.in_use.return_value = False
+         ent3.persisted.return_value = True
+         ent3.objsize.return_value = 600
+         ent3.clear.return_value = True
+ 
          # Delete ent1
+         self.assertEqual(500, cache.total())
          ent1.clear.return_value = True
 +        ent1.ref_count = 0
          inodes.del_entry(ent1)
          self.assertEqual(0, cache.total())
          cache.touch(ent3)