15295: Updates fuse code with futurize stage2 and fixes tests
authorEric Biagiotti <ebiagiotti@veritasgenetics.com>
Mon, 17 Jun 2019 21:34:16 +0000 (17:34 -0400)
committerEric Biagiotti <ebiagiotti@veritasgenetics.com>
Mon, 17 Jun 2019 21:34:16 +0000 (17:34 -0400)
Arvados-DCO-1.1-Signed-off-by: Eric Biagiotti <ebiagiotti@veritasgenetics.com>

services/fuse/arvados_fuse/__init__.py
services/fuse/arvados_fuse/command.py
services/fuse/arvados_fuse/crunchstat.py
services/fuse/arvados_fuse/fresh.py
services/fuse/arvados_fuse/fusedir.py
services/fuse/arvados_version.py
services/fuse/setup.py

index 72ccdb5f947c3baf4f99286faa1e18abd08ac470..f944a9197c8449af7431119816da575e20ab480b 100644 (file)
@@ -50,6 +50,15 @@ an object that is live in the inode cache, that object is immediately updated.
 """
 
 from __future__ import absolute_import
+from __future__ import division
+from future.utils import viewitems
+from future.utils import native
+from future import standard_library
+standard_library.install_aliases()
+from builtins import next
+from builtins import str
+from builtins import object
+from builtins import dict
 import os
 import sys
 import llfuse
@@ -73,8 +82,7 @@ import collections
 import functools
 import arvados.keep
 from prometheus_client import Summary
-
-import Queue
+import queue
 
 # Default _notify_queue has a limit of 1000 items, but it really needs to be
 # unlimited to avoid deadlocks, see https://arvados.org/issues/3198#note-43 for
@@ -82,10 +90,10 @@ import Queue
 
 if hasattr(llfuse, 'capi'):
     # llfuse < 0.42
-    llfuse.capi._notify_queue = Queue.Queue()
+    llfuse.capi._notify_queue = queue.Queue()
 else:
     # llfuse >= 0.42
-    llfuse._notify_queue = Queue.Queue()
+    llfuse._notify_queue = queue.Queue()
 
 LLFUSE_VERSION_0 = llfuse.__version__.startswith('0')
 
@@ -262,7 +270,7 @@ class Inodes(object):
         self._entries[key] = item
 
     def __iter__(self):
-        return self._entries.iterkeys()
+        return iter(self._entries.keys())
 
     def items(self):
         return self._entries.items()
@@ -303,12 +311,12 @@ class Inodes(object):
         if entry.has_ref(False):
             # Only necessary if the kernel has previously done a lookup on this
             # inode and hasn't yet forgotten about it.
-            llfuse.invalidate_entry(entry.inode, name.encode(self.encoding))
+            llfuse.invalidate_entry(entry.inode, native(name.encode(self.encoding)))
 
     def clear(self):
         self.inode_cache.clear()
 
-        for k,v in self._entries.items():
+        for k,v in viewitems(self._entries):
             try:
                 v.finalize()
             except Exception as e:
@@ -526,7 +534,7 @@ class Operations(llfuse.Operations):
         entry.st_size = e.size()
 
         entry.st_blksize = 512
-        entry.st_blocks = (entry.st_size/512)+1
+        entry.st_blocks = (entry.st_size // 512) + 1
         if hasattr(entry, 'st_atime_ns'):
             # llfuse >= 0.42
             entry.st_atime_ns = int(e.atime() * 1000000000)
@@ -567,7 +575,7 @@ class Operations(llfuse.Operations):
     @lookup_time.time()
     @catch_exceptions
     def lookup(self, parent_inode, name, ctx=None):
-        name = unicode(name, self.inodes.encoding)
+        name = str(name, self.inodes.encoding)
         inode = None
 
         if name == '.':
index f174d1bb02c7ef1264a8d52a2079c11bb06778a7..14b1327d97747f2d9e514ce30d4b76a29999e249 100644 (file)
@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+from builtins import range
+from builtins import str
+from builtins import object
 import argparse
 import arvados
 import daemon
@@ -134,12 +137,12 @@ class Mount(object):
         if self.args.replace:
             unmount(path=self.args.mountpoint,
                     timeout=self.args.unmount_timeout)
-        llfuse.init(self.operations, self.args.mountpoint, self._fuse_options())
+        llfuse.init(self.operations, self.args.mountpoint.encode('utf-8'), self._fuse_options())
         if self.daemon:
             daemon.DaemonContext(
                 working_directory=os.path.dirname(self.args.mountpoint),
-                files_preserve=range(
-                    3, resource.getrlimit(resource.RLIMIT_NOFILE)[1])
+                files_preserve=list(range(
+                    3, resource.getrlimit(resource.RLIMIT_NOFILE)[1]))
             ).open()
         if self.listen_for_events and not self.args.disable_event_listening:
             self.operations.listen_for_events()
index e99573752e1c0e69be8f2e07201530bf56a4024b..0cb585a6ff6d551a7db86553282d47ae4fecdb60 100644 (file)
@@ -2,13 +2,15 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+from builtins import str
+from builtins import object
 import sys
 import time
 from collections import namedtuple
 
 Stat = namedtuple("Stat", ['name', 'get'])
 
-class StatWriter(object):    
+class StatWriter(object):
     def __init__(self, prefix, interval, stats):
         self.prefix = prefix
         self.interval = interval
@@ -40,7 +42,7 @@ class StatWriter(object):
 
 def statlogger(interval, keep, ops):
     calls = StatWriter("keepcalls", interval, [
-        Stat("put", keep.put_counter.get), 
+        Stat("put", keep.put_counter.get),
         Stat("get", keep.get_counter.get)
     ])
     net = StatWriter("net:keep0", interval, [
@@ -48,15 +50,15 @@ def statlogger(interval, keep, ops):
         Stat("rx", keep.download_counter.get)
     ])
     cache = StatWriter("keepcache", interval, [
-        Stat("hit", keep.hits_counter.get), 
+        Stat("hit", keep.hits_counter.get),
         Stat("miss", keep.misses_counter.get)
     ])
     fuseops = StatWriter("fuseops", interval, [
-        Stat("write", ops.write_ops_counter.get), 
+        Stat("write", ops.write_ops_counter.get),
         Stat("read", ops.read_ops_counter.get)
     ])
     fusetimes = []
-    for cur_op in ops.metric_op_names():   
+    for cur_op in ops.metric_op_names():
         name = "fuseop:{0}".format(cur_op)
         fusetimes.append(StatWriter(name, interval, [
             Stat("count", ops.metric_count_func(cur_op)),
index 2e7a2a8182b62e2ffe8aa474a75c1f35b2723a43..53214ee94d70b214f79e3cca5c5193a41ebe2567 100644 (file)
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+from builtins import object
 import time
 import ciso8601
 import calendar
index 75fa2762da4b54205cd1ed83f71b9ef93cd12d78..9d53df950b8cd4886398be7134d4207dcb0beef5 100644 (file)
@@ -3,6 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 from __future__ import absolute_import
+from __future__ import division
+from future.utils import viewitems
+from builtins import dict
 import logging
 import re
 import time
@@ -164,7 +167,7 @@ class Directory(FreshBase):
     def in_use(self):
         if super(Directory, self).in_use():
             return True
-        for v in self._entries.itervalues():
+        for v in self._entries.values():
             if v.in_use():
                 return True
         return False
@@ -172,7 +175,7 @@ class Directory(FreshBase):
     def has_ref(self, only_children):
         if super(Directory, self).has_ref(only_children):
             return True
-        for v in self._entries.itervalues():
+        for v in self._entries.values():
             if v.has_ref(False):
                 return True
         return False
@@ -194,7 +197,7 @@ class Directory(FreshBase):
         # Find self on the parent in order to invalidate this path.
         # Calling the public items() method might trigger a refresh,
         # which we definitely don't want, so read the internal dict directly.
-        for k,v in parent._entries.items():
+        for k,v in viewitems(parent._entries):
             if v is self:
                 self.inodes.invalidate_entry(parent, k)
                 break
@@ -283,7 +286,7 @@ class CollectionDirectoryBase(Directory):
     def populate(self, mtime):
         self._mtime = mtime
         self.collection.subscribe(self.on_event)
-        for entry, item in self.collection.items():
+        for entry, item in viewitems(self.collection):
             self.new_entry(entry, item, self.mtime())
 
     def writable(self):
@@ -360,7 +363,7 @@ class CollectionDirectory(CollectionDirectoryBase):
         self.collection_record = None
         self._poll = True
         try:
-            self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2)/2)
+            self._poll_time = (api._rootDesc.get('blobSignatureTtl', 60*60*2) // 2)
         except:
             _logger.debug("Error getting blobSignatureTtl from discovery document: %s", sys.exc_info()[0])
             self._poll_time = 60*60
@@ -1117,7 +1120,7 @@ class SharedDirectory(Directory):
 
             # end with llfuse.lock_released, re-acquire lock
 
-            self.merge(contents.items(),
+            self.merge(viewitems(contents),
                        lambda i: i[0],
                        lambda a, i: a.uuid() == i[1]['uuid'],
                        lambda i: ProjectDirectory(self.inode, self.inodes, self.api, self.num_retries, i[1], poll=self._poll, poll_time=self._poll_time))
index 2e6484cabdf1e71d39f5fe21139b29c2ce09ad93..4253a09f67f7dbca02e46ff95b84533a2b7c43d6 100644 (file)
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from builtins import str
+from builtins import next
 import subprocess
 import time
 import os
index cbc9cb23edf5f36f0c6a83120d8c21cc0f582241..28f0c54175922105b597ac75ab45dbfd8f9149df 100644 (file)
@@ -39,8 +39,7 @@ setup(name='arvados_fuse',
       ],
       install_requires=[
         'arvados-python-client >= 0.1.20151118035730',
-        # llfuse 1.3.4 fails to install via pip
-        'llfuse >=1.2, <1.3.4',
+        'llfuse >= 1.3.6',
         'python-daemon',
         'ciso8601 >= 2.0.0',
         'setuptools',