Merge branch '8784-dir-listings'

[arvados.git] / sdk / python / arvados / cache.py

diff --git a/sdk/python/arvados/cache.py b/sdk/python/arvados/cache.py
index 6ab6b16eb9875bce7cb9b0d447a2f6fa876336fd..0c5061bb1acbd562c408015a82c30e7181239405 100644 (file)
--- a/sdk/python/arvados/cache.py
+++ b/sdk/python/arvados/cache.py
@@ -1,24 +1,46 @@
+from builtins import object
 import errno
-import md5
+import hashlib
 import os
 import tempfile
+import time
 
 class SafeHTTPCache(object):
-    def __init__(self, path=None):
+    """Thread-safe replacement for httplib2.FileCache"""
+
+    def __init__(self, path=None, max_age=None):
         self._dir = path
+        if max_age is not None:
+            try:
+                self._clean(threshold=time.time() - max_age)
+            except:
+                pass
+
+    def _clean(self, threshold=0):
+        for ent in os.listdir(self._dir):
+            fnm = os.path.join(self._dir, ent)
+            if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
+                continue
+            stat = os.lstat(fnm)
+            if stat.st_mtime < threshold:
+                try:
+                    os.unlink(fnm)
+                except OSError as err:
+                    if err.errno != errno.ENOENT:
+                        raise
 
     def __str__(self):
         return self._dir
 
     def _filename(self, url):
-        return os.path.join(self._dir, md5.new(url).hexdigest()+'.tmp')
+        return os.path.join(self._dir, hashlib.md5(url.encode('utf-8')).hexdigest()+'.tmp')
 
     def get(self, url):
         filename = self._filename(url)
         try:
             with open(filename, 'rb') as f:
                 return f.read()
-        except IOError, OSError:
+        except (IOError, OSError):
             return None
 
     def set(self, url, content):
@@ -28,7 +50,7 @@ class SafeHTTPCache(object):
             return None
         try:
             try:
-                f = os.fdopen(fd, 'w')
+                f = os.fdopen(fd, 'wb')
             except:
                 os.close(fd)
                 raise