projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
12183: Improve error handling if unable to delete symlinks marked for deletion.
[arvados.git]
/
crunch_scripts
/
crunchutil
/
robust_put.py
diff --git
a/crunch_scripts/crunchutil/robust_put.py
b/crunch_scripts/crunchutil/robust_put.py
index 4ea9d56ad004378a8599e09eeec15aebf83a0712..27b0bf34567b0489da7095be63f836344bb7b47c 100644
(file)
--- a/
crunch_scripts/crunchutil/robust_put.py
+++ b/
crunch_scripts/crunchutil/robust_put.py
@@
-1,3
+1,7
@@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados
import arvados.commands.put as put
import os
import arvados
import arvados.commands.put as put
import os
@@
-16,7
+20,10
@@
class Args(object):
# Upload to Keep with error recovery.
# Return a uuid or raise an exception if there are too many failures.
# Upload to Keep with error recovery.
# Return a uuid or raise an exception if there are too many failures.
-def upload(source_dir):
+def upload(source_dir, logger=None):
+ if logger is None:
+ logger = logging.getLogger("arvados")
+
source_dir = os.path.abspath(source_dir)
done = False
if 'TASK_WORK' in os.environ:
source_dir = os.path.abspath(source_dir)
done = False
if 'TASK_WORK' in os.environ:
@@
-35,15
+42,15
@@
def upload(source_dir):
outuuid = out.finish()
done = True
except KeyboardInterrupt as e:
outuuid = out.finish()
done = True
except KeyboardInterrupt as e:
- logg
ing
.critical("caught interrupt signal 2")
+ logg
er
.critical("caught interrupt signal 2")
raise e
except Exception as e:
raise e
except Exception as e:
- logg
ing
.exception("caught exception:")
+ logg
er
.exception("caught exception:")
backoff *= 2
if backoff > 256:
backoff *= 2
if backoff > 256:
- logg
ing
.critical("Too many upload failures, giving up")
+ logg
er
.critical("Too many upload failures, giving up")
raise e
else:
raise e
else:
- logg
ing
.warning("Sleeping for %s seconds before trying again" % backoff)
+ logg
er
.warning("Sleeping for %s seconds before trying again" % backoff)
time.sleep(backoff)
return outuuid
time.sleep(backoff)
return outuuid