+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
import arvados
import arvados.commands.put as put
import os
import logging
+import time
def machine_progress(bytes_written, bytes_expected):
return "upload wrote {} total {}\n".format(
# Upload to Keep with error recovery.
# Return a uuid or raise an exception if there are too many failures.
-def upload(source_dir):
+def upload(source_dir, logger=None):
+ if logger is None:
+ logger = logging.getLogger("arvados")
+
source_dir = os.path.abspath(source_dir)
done = False
if 'TASK_WORK' in os.environ:
outuuid = out.finish()
done = True
except KeyboardInterrupt as e:
- logging.critical("caught interrupt signal 2")
+ logger.critical("caught interrupt signal 2")
raise e
except Exception as e:
- logging.exception("caught exception:")
+ logger.exception("caught exception:")
backoff *= 2
if backoff > 256:
- logging.critical("Too many upload failures, giving up")
+ logger.critical("Too many upload failures, giving up")
raise e
else:
- logging.warning("Sleeping for %s seconds before trying again" % backoff)
+ logger.warning("Sleeping for %s seconds before trying again" % backoff)
time.sleep(backoff)
return outuuid