10111: Merge branch 'master' into 10111-cr-provenance-graph

[arvados.git] / sdk / python / arvados / cache.py

diff --git a/sdk/python/arvados/cache.py b/sdk/python/arvados/cache.py
index c4e4c09c62e33e9fdc85b11b7c23dd26ed60d072..08c19e4aa7de1423bf80122fe437a732980380d8 100644 (file)
--- a/sdk/python/arvados/cache.py
+++ b/sdk/python/arvados/cache.py
@@ -2,16 +2,37 @@ import errno
 import md5
 import os
 import tempfile
+import time
 
 class SafeHTTPCache(object):
-    def __init__(self, path=None):
+    """Thread-safe replacement for httplib2.FileCache"""
+
+    def __init__(self, path=None, max_age=None):
         self._dir = path
+        if max_age is not None:
+            try:
+                self._clean(threshold=time.time() - max_age)
+            except:
+                pass
+
+    def _clean(self, threshold=0):
+        for ent in os.listdir(self._dir):
+            fnm = os.path.join(self._dir, ent)
+            if os.path.isdir(fnm) or not fnm.endswith('.tmp'):
+                continue
+            stat = os.lstat(fnm)
+            if stat.st_mtime < threshold:
+                try:
+                    os.unlink(fnm)
+                except OSError as err:
+                    if err.errno != errno.ENOENT:
+                        raise
 
     def __str__(self):
         return self._dir
 
     def _filename(self, url):
-        return os.path.join(self._dir, md5.new(url).hexdigest())
+        return os.path.join(self._dir, md5.new(url).hexdigest()+'.tmp')
 
     def get(self, url):
         filename = self._filename(url)