projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
3147: Add retry support to PySDK list_all utility.
[arvados.git]
/
crunch_scripts
/
crunchutil
/
robust_put.py
diff --git
a/crunch_scripts/crunchutil/robust_put.py
b/crunch_scripts/crunchutil/robust_put.py
index 4ea9d56ad004378a8599e09eeec15aebf83a0712..5a6d59384fb4fc4e75453d81ba4077ea49f4db70 100644
(file)
--- a/
crunch_scripts/crunchutil/robust_put.py
+++ b/
crunch_scripts/crunchutil/robust_put.py
@@
-16,7
+16,10
@@
class Args(object):
# Upload to Keep with error recovery.
# Return a uuid or raise an exception if there are too many failures.
# Upload to Keep with error recovery.
# Return a uuid or raise an exception if there are too many failures.
-def upload(source_dir):
+def upload(source_dir, logger=None):
+ if logger is None:
+ logger = logging.getLogger("arvados")
+
source_dir = os.path.abspath(source_dir)
done = False
if 'TASK_WORK' in os.environ:
source_dir = os.path.abspath(source_dir)
done = False
if 'TASK_WORK' in os.environ:
@@
-35,15
+38,15
@@
def upload(source_dir):
outuuid = out.finish()
done = True
except KeyboardInterrupt as e:
outuuid = out.finish()
done = True
except KeyboardInterrupt as e:
- logg
ing
.critical("caught interrupt signal 2")
+ logg
er
.critical("caught interrupt signal 2")
raise e
except Exception as e:
raise e
except Exception as e:
- logg
ing
.exception("caught exception:")
+ logg
er
.exception("caught exception:")
backoff *= 2
if backoff > 256:
backoff *= 2
if backoff > 256:
- logg
ing
.critical("Too many upload failures, giving up")
+ logg
er
.critical("Too many upload failures, giving up")
raise e
else:
raise e
else:
- logg
ing
.warning("Sleeping for %s seconds before trying again" % backoff)
+ logg
er
.warning("Sleeping for %s seconds before trying again" % backoff)
time.sleep(backoff)
return outuuid
time.sleep(backoff)
return outuuid