From f70c457c73f1bff6314f8c3eb4ad7be0c44a5f1a Mon Sep 17 00:00:00 2001 From: Lucas Di Pentima Date: Wed, 12 Apr 2017 13:07:05 -0300 Subject: [PATCH] 11341: Use NamedTemporaryFile instead of mkstemp for cache saving. Added logging messages to inform the user what's happening with the cache file. --- sdk/python/arvados/commands/put.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/sdk/python/arvados/commands/put.py b/sdk/python/arvados/commands/put.py index 32d5fef6a8..83fde9cba4 100644 --- a/sdk/python/arvados/commands/put.py +++ b/sdk/python/arvados/commands/put.py @@ -543,7 +543,10 @@ class ArvPutUploadJob(object): with self._state_lock: self._state['manifest'] = manifest if self.use_cache: - self._save_state() + try: + self._save_state() + except Exception as e: + self.logger.error("Unexpected error trying to save cache file: {}".format(e)) else: self.bytes_written = self.bytes_skipped # Call the reporter, if any @@ -673,10 +676,12 @@ class ArvPutUploadJob(object): cache_filepath = os.path.join( arv_cmd.make_home_conf_dir(self.CACHE_DIR, 0o700, 'raise'), cache_filename) - if self.resume: + if self.resume and os.path.exists(cache_filepath): + self.logger.info("Resuming upload from cache file {}".format(cache_filepath)) self._cache_file = open(cache_filepath, 'a+') else: # --no-resume means start with a empty cache file. + self.logger.info("Creating new cache file create at {}".format(cache_filepath)) self._cache_file = open(cache_filepath, 'w+') self._cache_filename = self._cache_file.name self._lock_file(self._cache_file) @@ -719,20 +724,19 @@ class ArvPutUploadJob(object): """ Atomically save current state into cache. """ + with self._state_lock: + # We're not using copy.deepcopy() here because it's a lot slower + # than json.dumps(), and we're already needing JSON format to be + # saved on disk. + state = json.dumps(self._state) try: - with self._state_lock: - # We're not using copy.deepcopy() here because it's a lot slower - # than json.dumps(), and we're already needing JSON format to be - # saved on disk. - state = json.dumps(self._state) - new_cache_fd, new_cache_name = tempfile.mkstemp( - dir=os.path.dirname(self._cache_filename)) - self._lock_file(new_cache_fd) - new_cache = os.fdopen(new_cache_fd, 'r+') + new_cache = tempfile.NamedTemporaryFile( + dir=os.path.dirname(self._cache_filename), delete=False) + self._lock_file(new_cache) new_cache.write(state) new_cache.flush() os.fsync(new_cache) - os.rename(new_cache_name, self._cache_filename) + os.rename(new_cache.name, self._cache_filename) except (IOError, OSError, ResumeCacheConflict) as error: self.logger.error("There was a problem while saving the cache file: {}".format(error)) try: -- 2.30.2